From b5c3969f0b5fa2dba7675f404e01e22963942cf8 Mon Sep 17 00:00:00 2001 From: Stephen Jia Date: Mon, 12 Oct 2026 12:45:45 -0700 Subject: [PATCH 1/7] [ET-VK][test] Allow FP16 underflow in SwiGLU comparisons Pull Request resolved: https://github.com/pytorch/executorch/pull/23716 ## Ulterior Motive Keep `VulkanSwiGLUTest` correctness checks usable on Vulkan devices that flush FP16 subnormal values to zero. ## Rationale **What**: Increase the absolute error allowances in `MatchesUnfusedAcrossResizes` to account for FP16 underflow. **Why**: On stable, all 28 FP16 configurations fail on Adreno 750 while FP32 passes. The first expected output is `5.30033e-6`, below FP16's minimum normal value, but the shader returns zero. The unfused path can also flush subnormal inputs and intermediate tensors before later multiplications amplify the difference. ## Details - Fused versus CPU reference: - **Before**: Absolute allowance `1e-7`. - **After**: Absolute allowance `2^-14`, FP16's minimum normal value; relative allowance remains `1e-3`. - Fused versus unfused graph: - **Before**: Absolute allowance `6e-8 * (abs(x*y) + abs(y) + 1)` assumes subnormal preservation. - **After**: Keep the original rounding allowance and add `(1 + 1e-3) * abs(reference)` only when an FP16 input is subnormal or a computed intermediate/output can round below the minimum normal value. Use the existing `3e-3` relative tolerance as the rounding margin at that boundary. - If the unfused result flushes to zero, its relative tolerance vanishes. `abs(reference)` covers the lost output, and the extra `1e-3` covers the fused result's relative error. - Cases without underflow keep the small original absolute allowance: `x=20, y=2` gives `2.58e-6`; the original tiny-output case gives about `6.63e-6`. - FP32 uses its existing assertions. Both FP16 comparisons keep `ASSERT_NEAR`. Differential Revision: [D124833698](https://our.internmc.facebook.com/intern/diff/D124833698/) ghstack-source-id: 446951678 --- .../vulkan/test/vulkan_compute_api_test.cpp | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/backends/vulkan/test/vulkan_compute_api_test.cpp b/backends/vulkan/test/vulkan_compute_api_test.cpp index d77b8e8c95c..d56fee3713a 100644 --- a/backends/vulkan/test/vulkan_compute_api_test.cpp +++ b/backends/vulkan/test/vulkan_compute_api_test.cpp @@ -4316,15 +4316,35 @@ TEST_P(VulkanSwiGLUTest, MatchesUnfusedAcrossResizes) { const float y = float(up_data[broadcast ? i % width : i]); const float reference = (x / (1.0f + std::exp(-x))) * y; if (dtype == vkapi::kHalf) { + constexpr float fp16_min_normal = 0x1p-14f; + constexpr float unfused_rtol = 3e-3f; ASSERT_NEAR( - float(actual[i]), reference, 1e-3f * std::abs(reference) + 1e-7f) + float(actual[i]), + reference, + 1e-3f * std::abs(reference) + fp16_min_normal) << "element " << i; - // The unfused path rounds twice more, including subnormal sigmoid. - const float atol = 6e-8f * (std::abs(x * y) + std::abs(y) + 1.0f); + const float sigmoid_reference = 1.0f / (1.0f + std::exp(-x)); + const auto is_subnormal = [](const float value) { + return value != 0.0f && std::abs(value) < fp16_min_normal; + }; + const auto may_underflow = [](const float value) { + return value != 0.0f && + std::abs(value) < fp16_min_normal * (1.0f + unfused_rtol); + }; + float atol = 6e-8f * (std::abs(x * y) + std::abs(y) + 1.0f); + if (is_subnormal(x) || is_subnormal(y) || + may_underflow(sigmoid_reference) || + may_underflow(x * sigmoid_reference) || + may_underflow(reference)) { + // Flushing the unfused path to zero can lose the full reference + // magnitude. Its relative tolerance then vanishes, so the extra + // 1e-3 covers the fused result's relative error. + atol += (1.0f + 1e-3f) * std::abs(reference); + } ASSERT_NEAR( float(actual[i]), float(expected[i]), - 3e-3f * std::abs(float(expected[i])) + atol) + unfused_rtol * std::abs(float(expected[i])) + atol) << "element " << i; } else { ASSERT_NEAR( From 72c78ae1a0f658116d43f42af5fa902a873db669 Mon Sep 17 00:00:00 2001 From: Stephen Jia Date: Mon, 12 Oct 2026 12:45:45 -0700 Subject: [PATCH 2/7] [ET-VK][codegen] Support nested shader directories Pull Request resolved: https://github.com/pytorch/executorch/pull/23490 ## Ulterior Motive Reorganize the flat Vulkan shader directory by operator implementation route before landing additional convolution implementations such as Winograd. ## Rationale **What**: Make shader code generation and build discovery preserve nested source paths. **Why**: The generator discovered nested files but identified and emitted them by basename. Repeated basenames overwrote each other, YAML templates were globally keyed, generated files were flattened, and build inputs excluded nested files. ## Details - Shader identity - **Before**: Source, YAML template, generated source, cache, and SPIR-V paths were basename-based. - **After**: Paths are relative to each shader source root and preserve directories through generation and caching. - Includes - **Before**: Cache invalidation resolved includes only from the generated output root. - **After**: Includes resolve from the including shader's directory, then each ancestor directory up to the generated output root, nearest first. `glslc` receives the same include path list, so route subdirectories (e.g. `convolution/conv2d/im2col/`) can share family-level headers (e.g. `convolution/conv2d/conv2d_common.glslh`). - Build discovery - **Before**: Buck and CMake shader filegroups selected only direct children. - **After**: Runtime, test, custom-op, and tool shader filegroups recursively include nested files. Runtime shader registry names remain flat and unchanged. Duplicate runtime shader names now fail during code generation instead of silently overwriting an earlier source. ghstack-source-id: 446951693 @exported-using-ghexport Differential Revision: [D123101914](https://our.internmc.facebook.com/intern/diff/D123101914/) --- backends/vulkan/cmake/ShaderLibrary.cmake | 10 +- backends/vulkan/runtime/gen_vulkan_spv.py | 108 ++++++-- backends/vulkan/targets.bzl | 2 +- backends/vulkan/test/custom_ops/targets.bzl | 3 +- .../test/spv_codegen/test_gen_vulkan_spv.py | 262 ++++++++++++++++++ backends/vulkan/test/targets.bzl | 12 +- backends/vulkan/tools/gpuinfo/targets.bzl | 2 +- 7 files changed, 372 insertions(+), 27 deletions(-) create mode 100644 backends/vulkan/test/spv_codegen/test_gen_vulkan_spv.py diff --git a/backends/vulkan/cmake/ShaderLibrary.cmake b/backends/vulkan/cmake/ShaderLibrary.cmake index d7e55cd3592..53d2839a45e 100644 --- a/backends/vulkan/cmake/ShaderLibrary.cmake +++ b/backends/vulkan/cmake/ShaderLibrary.cmake @@ -81,8 +81,14 @@ function(gen_vulkan_shader_lib_cpp shaders_path) endif() # Ninja cannot expand wildcards (*) in DEPENDS lists. - file(GLOB VULKAN_SHADERS "${shaders_path}/*.glsl" "${shaders_path}/*.glslh" - "${shaders_path}/*.yaml" "${shaders_path}/*.h" + file( + GLOB_RECURSE + VULKAN_SHADERS + CONFIGURE_DEPENDS + "${shaders_path}/*.glsl" + "${shaders_path}/*.glslh" + "${shaders_path}/*.yaml" + "${shaders_path}/*.h" ) add_custom_command( diff --git a/backends/vulkan/runtime/gen_vulkan_spv.py b/backends/vulkan/runtime/gen_vulkan_spv.py index c9b054f1e73..5016e9c48db 100644 --- a/backends/vulkan/runtime/gen_vulkan_spv.py +++ b/backends/vulkan/runtime/gen_vulkan_spv.py @@ -748,6 +748,7 @@ def __init__( self.src_files: Dict[str, str] = {} self.template_yaml_files: List[str] = [] + self.yaml_relative_dirs: Dict[str, str] = {} self.addSrcAndYamlFiles(self.src_dir_paths) self.shader_template_params: Dict[Any, Any] = {} @@ -759,20 +760,30 @@ def __init__( def addSrcAndYamlFiles(self, src_dir_paths: List[str]) -> None: for src_path in src_dir_paths: + src_path = os.path.abspath(src_path) # Collect glsl source files src_files_list = glob.glob( os.path.join(src_path, "**", "*.[gh]lsl*"), recursive=True ) + glob.glob(os.path.join(src_path, "**", "*.h"), recursive=True) - for file in src_files_list: + for file in sorted(src_files_list): if len(file) > 1: - self.src_files[extract_filename(file, keep_ext=False)] = file + relative_path = os.path.relpath(file, src_path) + src_name = os.path.join( + os.path.dirname(relative_path), + extract_filename(relative_path, keep_ext=False), + ) + if src_name in self.src_files: + raise KeyError(f"{relative_path} source file is defined twice") + self.src_files[src_name] = file # Collect template yaml files yaml_files = glob.glob( os.path.join(src_path, "**", "*.yaml"), recursive=True ) - for file in yaml_files: + for file in sorted(yaml_files): if len(file) > 1: self.template_yaml_files.append(file) + relative_path = os.path.relpath(file, src_path) + self.yaml_relative_dirs[file] = os.path.dirname(relative_path) def generateVariantCombinations( # noqa: C901 self, @@ -839,14 +850,17 @@ def parseTemplateYaml(self, yaml_file: str) -> None: # noqa: C901 with open(yaml_file) as f: contents = yaml.load(f, Loader=UniqueKeyLoader) for template_name, params_dict in contents.items(): - if template_name in self.shader_template_params: - raise KeyError(f"{template_name} params file is defined twice") + template_path = os.path.normpath( + os.path.join(self.yaml_relative_dirs[yaml_file], template_name) + ) + if template_path in self.shader_template_params: + raise KeyError(f"{template_path} params file is defined twice") default_params = params_dict["parameter_names_with_default_values"] default_params["YAML_SRC_FULLPATH"] = yaml_file params_names = set(default_params.keys()).union({"NAME"}) - self.shader_template_params[template_name] = [] + self.shader_template_params[template_path] = [] default_iterated_params = params_dict.get( "generate_variant_forall", None @@ -907,7 +921,7 @@ def parseTemplateYaml(self, yaml_file: str) -> None: # noqa: C901 default_params_copy["NAME"] = variant_name default_params_copy["VARIANT_NAME"] = variant["NAME"] - self.shader_template_params[template_name].append( + self.shader_template_params[template_path].append( default_params_copy ) else: @@ -916,7 +930,7 @@ def parseTemplateYaml(self, yaml_file: str) -> None: # noqa: C901 if key not in reserved_yaml_keys: default_params_copy[key] = variant[key] - self.shader_template_params[template_name].append( + self.shader_template_params[template_path].append( default_params_copy ) @@ -944,18 +958,36 @@ def create_shader_params( return shader_params def constructOutputMap(self) -> None: + shader_names = set() + + def add_output( + output_name: str, + src_file_fullpath: str, + params: Dict[str, str], + ) -> None: + if output_name in self.output_file_map: + raise KeyError(f"{output_name} output file is defined twice") + if extract_extension(src_file_fullpath) == "glsl": + shader_name = os.path.basename(output_name) + if shader_name in shader_names: + raise KeyError(f"{shader_name} shader is defined twice") + shader_names.add(shader_name) + self.output_file_map[output_name] = (src_file_fullpath, params) + for src_name, params in self.shader_template_params.items(): for variant in params: src_file_fullpath = self.src_files[src_name] - - self.output_file_map[variant["NAME"]] = ( + output_name = os.path.join(os.path.dirname(src_name), variant["NAME"]) + add_output( + output_name, src_file_fullpath, self.create_shader_params(variant), ) for src_name, src_file_fullpath in self.src_files.items(): if src_name not in self.shader_template_params: - self.output_file_map[src_name] = ( + add_output( + src_name, src_file_fullpath, self.create_shader_params(), ) @@ -1054,21 +1086,55 @@ def file_has_changed(gen_file_path, cached_file_path): cached_checksum = self.get_md5_checksum(cached_file_path) return current_checksum != cached_checksum - def any_sources_changed(gen_file_path, output_dir): + def get_include_dirs(gen_file_path): + """ + Return the directories searched for a generated file's includes: its + own directory, then each ancestor up to output_dir, nearest first. + """ + include_dir = os.path.dirname(gen_file_path) + include_dirs = [include_dir] + while include_dir != output_dir: + parent_dir = os.path.dirname(include_dir) + if parent_dir == include_dir: + raise ValueError( + f"{gen_file_path} is outside output directory {output_dir}" + ) + include_dir = parent_dir + include_dirs.append(include_dir) + return include_dirs + + def any_sources_changed(gen_file_path, visited=None): """ Given the path to a generated source file, check the gen_file_meta dict to determine if the ANY of the source files contributing to the compilation of this file were changed since the last successful compilation. """ + if visited is None: + visited = set() + if gen_file_path in visited: + return False + if gen_file_path not in gen_file_meta: + return True + visited.add(gen_file_path) + gen_file_changed, includes_list = gen_file_meta[gen_file_path] - any_changed = gen_file_changed + if gen_file_changed: + return True + include_dirs = get_include_dirs(gen_file_path) for included_file in includes_list: - included_file_path = os.path.join(output_dir, included_file) - any_changed = any_changed or any_sources_changed( - included_file_path, output_dir + candidate_paths = [ + os.path.normpath(os.path.join(include_dir, included_file)) + for include_dir in include_dirs + ] + included_file_path = next( + (path for path in candidate_paths if path in gen_file_meta), + # Untracked includes must disable the cache. + candidate_paths[-1], ) + if any_sources_changed(included_file_path, visited): + return True - return any_changed + return False def generate_src_file(shader_paths_pair) -> Tuple[bool, List[str]]: """ @@ -1102,6 +1168,8 @@ def generate_src_file(shader_paths_pair) -> Tuple[bool, List[str]]: cached_gen_out_path = os.path.join( cache_dir, f"{src_file_name}.{out_file_ext}" ) + os.makedirs(os.path.dirname(gen_out_path), exist_ok=True) + os.makedirs(os.path.dirname(cached_gen_out_path), exist_ok=True) # Execute codegen to generate the output file with codecs.open(template_file_path, "r", encoding="utf-8") as input_file: @@ -1191,7 +1259,7 @@ def compile_spirv(shader_paths_pair) -> Tuple[str, str]: # in the cache. cached_spv_out_path = os.path.join(cache_dir, f"{src_file_name}.spv") - can_use_cached = not any_sources_changed(gen_out_path, output_dir) + can_use_cached = not any_sources_changed(gen_out_path) if can_use_cached and os.path.exists(cached_spv_out_path): shutil.copyfile(cached_spv_out_path, spv_out_path) return (spv_out_path, gen_out_path) @@ -1207,9 +1275,9 @@ def compile_spirv(shader_paths_pair) -> Tuple[str, str]: spv_out_path, "--target-env=vulkan{}".format(vk_version), "-Werror", - "-I", - output_dir, ] + for include_dir in get_include_dirs(gen_out_path): + cmd_base += ["-I", include_dir] cmd = cmd_base + self.glslc_flags try: diff --git a/backends/vulkan/targets.bzl b/backends/vulkan/targets.bzl index d2dee601399..aaa96eccb63 100644 --- a/backends/vulkan/targets.bzl +++ b/backends/vulkan/targets.bzl @@ -184,7 +184,7 @@ def define_common_targets(is_fbcode = False): runtime.filegroup( name = "vulkan_graph_runtime_shaders", srcs = native.glob([ - "runtime/graph/ops/glsl/*", + "runtime/graph/ops/glsl/**/*", ]), ) diff --git a/backends/vulkan/test/custom_ops/targets.bzl b/backends/vulkan/test/custom_ops/targets.bzl index 1895180495a..e3631298119 100644 --- a/backends/vulkan/test/custom_ops/targets.bzl +++ b/backends/vulkan/test/custom_ops/targets.bzl @@ -34,8 +34,7 @@ def define_common_targets(is_fbcode = False): runtime.filegroup( name = "custom_ops_shaders", srcs = native.glob([ - "glsl/*.glsl", - "glsl/*.yaml", + "glsl/**/*", ]), visibility = ["PUBLIC"], ) diff --git a/backends/vulkan/test/spv_codegen/test_gen_vulkan_spv.py b/backends/vulkan/test/spv_codegen/test_gen_vulkan_spv.py new file mode 100644 index 00000000000..5a5d78ee5b5 --- /dev/null +++ b/backends/vulkan/test/spv_codegen/test_gen_vulkan_spv.py @@ -0,0 +1,262 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import stat +import tempfile +import unittest +from pathlib import Path + +from runtime import gen_vulkan_spv + + +class TestNestedShaderSources(unittest.TestCase): + def _write_fake_compiler( + self, directory: Path, assertions: str = "" + ) -> tuple[Path, Path]: + compiler_log = directory / "compiler.log" + compiler = directory / "fake_glslc.py" + compiler.write_text( + "#!/usr/bin/env python3\n" + "import pathlib\n" + "import sys\n" + "log = pathlib.Path(__file__).with_name('compiler.log')\n" + "with log.open('a') as log_file:\n" + " log_file.write('compile\\n')\n" + "source = pathlib.Path(sys.argv[2])\n" + + assertions + + "output = pathlib.Path(sys.argv[sys.argv.index('-o') + 1])\n" + "output.write_bytes(b'\\x03\\x02\\x23\\x07')\n" + ) + compiler.chmod(compiler.stat().st_mode | stat.S_IXUSR) + return compiler, compiler_log + + def _write_shader(self, directory: Path, variant_name: str) -> None: + directory.mkdir(parents=True) + (directory / "common.glslh").write_text("const uint value = 1;\n") + (directory / "kernel.glsl").write_text( + '#version 450\n#include "common.glslh"\nvoid main() {}\n' + ) + (directory / "kernel.yaml").write_text( + "kernel:\n" + " parameter_names_with_default_values: {}\n" + " shader_variants:\n" + f" - NAME: {variant_name}\n" + ) + + def test_repeated_template_names_in_nested_directories(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + self._write_shader(source_root / "direct", "direct_kernel") + self._write_shader(source_root / "winograd", "winograd_kernel") + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=None + ) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + self.assertTrue((output_dir / "direct" / "direct_kernel.glsl").is_file()) + self.assertTrue((output_dir / "direct" / "common.glslh").is_file()) + self.assertTrue( + (output_dir / "winograd" / "winograd_kernel.glsl").is_file() + ) + self.assertTrue((output_dir / "winograd" / "common.glslh").is_file()) + + def test_multi_dot_source_preserves_legacy_template_name(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + source_root.mkdir() + (source_root / "kernel.special.glsl").write_text( + "#version 450\nvoid main() {}\n" + ) + (source_root / "kernel.yaml").write_text( + "kernel:\n" + " parameter_names_with_default_values: {}\n" + " shader_variants:\n" + " - NAME: generated_kernel\n" + ) + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=None + ) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + self.assertTrue((output_dir / "generated_kernel.glsl").is_file()) + + def test_rejects_generated_shader_outside_output_directory(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + self._write_shader(source_root, str(root / "escaped_kernel")) + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + compiler, _ = self._write_fake_compiler(root) + + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=str(compiler) + ) + with self.assertRaisesRegex(ValueError, "outside output directory"): + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + def test_local_include_change_recompiles_nested_shader(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + shader_dir = source_root / "nested" + self._write_shader(shader_dir, "nested_kernel") + (source_root / "root_common.glslh").write_text( + "const uint root_value = 1;\n" + ) + (shader_dir / "kernel.glsl").write_text( + '#version 450\n#include "common.glslh"\n' + '#include "root_common.glslh"\nvoid main() {}\n' + ) + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + + compiler, compiler_log = self._write_fake_compiler( + root, + "include_indices = [i for i, arg in enumerate(sys.argv) if arg == '-I']\n" + "local_include_dir = pathlib.Path(sys.argv[include_indices[0] + 1])\n" + "root_include_dir = pathlib.Path(sys.argv[include_indices[1] + 1])\n" + "assert local_include_dir == source.parent\n" + "assert (local_include_dir / 'common.glslh').is_file()\n" + "assert root_include_dir == source.parent.parent\n" + "assert (root_include_dir / 'root_common.glslh').is_file()\n", + ) + + def generate() -> None: + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=str(compiler) + ) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + generate() + generate() + self.assertEqual(compiler_log.read_text().splitlines(), ["compile"]) + + (shader_dir / "common.glslh").write_text("const uint value = 2;\n") + generate() + + self.assertEqual( + compiler_log.read_text().splitlines(), ["compile", "compile"] + ) + + (source_root / "root_common.glslh").write_text( + "const uint root_value = 2;\n" + ) + generate() + + self.assertEqual( + compiler_log.read_text().splitlines(), + ["compile", "compile", "compile"], + ) + + def test_ancestor_include_change_recompiles_nested_shader(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + family_dir = source_root / "family" + shader_dir = family_dir / "route" + self._write_shader(shader_dir, "route_kernel") + (family_dir / "family_common.glslh").write_text( + "const uint family_value = 1;\n" + ) + (shader_dir / "kernel.glsl").write_text( + '#version 450\n#include "family_common.glslh"\nvoid main() {}\n' + ) + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + + compiler, compiler_log = self._write_fake_compiler( + root, + "include_dirs = [pathlib.Path(sys.argv[i + 1]) " + "for i, arg in enumerate(sys.argv) if arg == '-I']\n" + "assert include_dirs == [source.parent, source.parent.parent, " + "source.parent.parent.parent]\n", + ) + + def generate() -> None: + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=str(compiler) + ) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + generate() + generate() + self.assertEqual(compiler_log.read_text().splitlines(), ["compile"]) + + (family_dir / "family_common.glslh").write_text( + "const uint family_value = 2;\n" + ) + generate() + + self.assertEqual( + compiler_log.read_text().splitlines(), ["compile", "compile"] + ) + + def test_untracked_include_disables_cache(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + self._write_shader(source_root, "kernel") + (source_root / "kernel.glsl").write_text( + '#version 450\n#include "external.glslh"\nvoid main() {}\n' + ) + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + compiler, compiler_log = self._write_fake_compiler(root) + + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=str(compiler) + ) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + self.assertEqual( + compiler_log.read_text().splitlines(), ["compile", "compile"] + ) + + def test_cyclic_includes_use_cache(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + self._write_shader(source_root, "kernel") + (source_root / "kernel.glsl").write_text( + '#version 450\n#include "a.glslh"\nvoid main() {}\n' + ) + (source_root / "a.glslh").write_text('#include "b.glslh"\n') + (source_root / "b.glslh").write_text('#include "a.glslh"\n') + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + compiler, compiler_log = self._write_fake_compiler(root) + + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=str(compiler) + ) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + self.assertEqual(compiler_log.read_text().splitlines(), ["compile"]) diff --git a/backends/vulkan/test/targets.bzl b/backends/vulkan/test/targets.bzl index 0051a74a530..e3b735137d5 100644 --- a/backends/vulkan/test/targets.bzl +++ b/backends/vulkan/test/targets.bzl @@ -98,6 +98,16 @@ def define_common_targets(is_fbcode = False): ], ) + python_unittest( + name = "test_gen_vulkan_spv", + srcs = [ + "spv_codegen/test_gen_vulkan_spv.py", + ], + deps = [ + "//executorch/backends/vulkan:gen_vulkan_spv_lib", + ], + ) + python_unittest( name = "test_serialization", srcs = [ @@ -164,7 +174,7 @@ def define_common_targets(is_fbcode = False): fb_native.filegroup( name = "test_shaders", srcs = glob([ - "glsl/*", + "glsl/**/*", ]), visibility = [ "PUBLIC", diff --git a/backends/vulkan/tools/gpuinfo/targets.bzl b/backends/vulkan/tools/gpuinfo/targets.bzl index 2bc9b3f1d04..be0a9680609 100644 --- a/backends/vulkan/tools/gpuinfo/targets.bzl +++ b/backends/vulkan/tools/gpuinfo/targets.bzl @@ -8,7 +8,7 @@ def define_common_targets(is_fbcode = False): buck_filegroup( name = "gpuinfo_shaders", srcs = native.glob([ - "glsl/*", + "glsl/**/*", ]), visibility = [ "PUBLIC", From ebcfcaf871c7092f4f6eb39c5555c1f2db00377a Mon Sep 17 00:00:00 2001 From: Stephen Jia Date: Mon, 12 Oct 2026 12:45:46 -0700 Subject: [PATCH 3/7] [ET-VK][binary] Nest binary operator sources Pull Request resolved: https://github.com/pytorch/executorch/pull/23491 ## Ulterior Motive Organize Vulkan operators by implementation family so operators with multiple routes can use clear, scalable directory layouts. ## Rationale **What**: Move standard binary GLSL, YAML, shared shader definitions, and C++ implementations into dedicated `binary/` directories. **Why**: The flat shader and implementation directories obscure ownership as operator families grow. Binary operators provide a contained validation of nested-source support before reorganizing larger families such as convolution. ## Details - Move tensor-tensor and tensor-scalar shader templates, YAML variant definitions, and `binary_op_defs.glslh` under `glsl/binary/`, and `q8ta_binary` under `glsl/binary/q8ta/`. - Move `BinaryOp.cpp` and `BinaryScalarOp.cpp` under `impl/binary/`, and the quantized `Q8taBinary.{h,cpp}` under `impl/binary/q8ta/`. - Update shader-name validation to address path-qualified template identities, the `Q8taBinary.h` include in `TestQ8taBinary.cpp`, and the binary section labels in `op_registry.py`. - Moved files are recorded as renames. - Preserve generated shader names and runtime dispatch behavior. ghstack-source-id: 446951706 @exported-using-ghexport Differential Revision: [D123101915](https://our.internmc.facebook.com/intern/diff/D123101915/) --- backends/vulkan/op_registry.py | 8 ++++---- .../ops/glsl/{ => binary}/binary_op_buffer.glsl | 0 .../ops/glsl/{ => binary}/binary_op_buffer.yaml | 0 .../graph/ops/glsl/{ => binary}/binary_op_defs.glslh | 0 .../ops/glsl/{ => binary}/binary_op_texture.glsl | 0 .../ops/glsl/{ => binary}/binary_op_texture.yaml | 0 .../ops/glsl/{ => binary}/binary_scalar_buffer.glsl | 0 .../ops/glsl/{ => binary}/binary_scalar_buffer.yaml | 0 .../ops/glsl/{ => binary}/binary_scalar_texture.glsl | 0 .../ops/glsl/{ => binary}/binary_scalar_texture.yaml | 0 .../ops/glsl/{ => binary/q8ta}/q8ta_binary.glsl | 0 .../ops/glsl/{ => binary/q8ta}/q8ta_binary.yaml | 0 .../runtime/graph/ops/impl/{ => binary}/BinaryOp.cpp | 0 .../graph/ops/impl/{ => binary}/BinaryScalarOp.cpp | 0 .../graph/ops/impl/{ => binary/q8ta}/Q8taBinary.cpp | 0 .../graph/ops/impl/{ => binary/q8ta}/Q8taBinary.h | 0 .../vulkan/test/custom_ops/impl/TestQ8taBinary.cpp | 2 +- backends/vulkan/test/test_vulkan_shader_names.py | 12 +++++++++--- 18 files changed, 14 insertions(+), 8 deletions(-) rename backends/vulkan/runtime/graph/ops/glsl/{ => binary}/binary_op_buffer.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => binary}/binary_op_buffer.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => binary}/binary_op_defs.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => binary}/binary_op_texture.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => binary}/binary_op_texture.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => binary}/binary_scalar_buffer.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => binary}/binary_scalar_buffer.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => binary}/binary_scalar_texture.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => binary}/binary_scalar_texture.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => binary/q8ta}/q8ta_binary.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => binary/q8ta}/q8ta_binary.yaml (100%) rename backends/vulkan/runtime/graph/ops/impl/{ => binary}/BinaryOp.cpp (100%) rename backends/vulkan/runtime/graph/ops/impl/{ => binary}/BinaryScalarOp.cpp (100%) rename backends/vulkan/runtime/graph/ops/impl/{ => binary/q8ta}/Q8taBinary.cpp (100%) rename backends/vulkan/runtime/graph/ops/impl/{ => binary/q8ta}/Q8taBinary.h (100%) diff --git a/backends/vulkan/op_registry.py b/backends/vulkan/op_registry.py index 6f4cd9b8978..461aeeaafcc 100644 --- a/backends/vulkan/op_registry.py +++ b/backends/vulkan/op_registry.py @@ -248,7 +248,7 @@ def register_clamp(): # ============================================================================= -# BinaryOp.cpp +# binary/BinaryOp.cpp # ============================================================================= @@ -302,7 +302,7 @@ def register_comparison_ops(): # ============================================================================= -# BinaryOp.cpp (bitwise) +# binary/BinaryOp.cpp (bitwise) # ============================================================================= @@ -325,7 +325,7 @@ def register_bool_binary_ops(): # ============================================================================= -# BinaryScalarOp.cpp +# binary/BinaryScalarOp.cpp # ============================================================================= @@ -627,7 +627,7 @@ def register_torchao_choose_qparams_affine(): # ============================================================================= -# Q8taBinary.cpp +# binary/q8ta/Q8taBinary.cpp # ============================================================================= diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_op_buffer.glsl b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_buffer.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_op_buffer.glsl rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_buffer.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_op_buffer.yaml b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_buffer.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_op_buffer.yaml rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_buffer.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_op_defs.glslh b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_defs.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_op_defs.glslh rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_defs.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_op_texture.glsl b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_texture.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_op_texture.glsl rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_texture.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_op_texture.yaml b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_texture.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_op_texture.yaml rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_texture.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_scalar_buffer.glsl b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_buffer.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_scalar_buffer.glsl rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_buffer.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_scalar_buffer.yaml b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_buffer.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_scalar_buffer.yaml rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_buffer.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_scalar_texture.glsl b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_texture.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_scalar_texture.glsl rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_texture.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_scalar_texture.yaml b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_texture.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_scalar_texture.yaml rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_texture.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_binary.glsl b/backends/vulkan/runtime/graph/ops/glsl/binary/q8ta/q8ta_binary.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_binary.glsl rename to backends/vulkan/runtime/graph/ops/glsl/binary/q8ta/q8ta_binary.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_binary.yaml b/backends/vulkan/runtime/graph/ops/glsl/binary/q8ta/q8ta_binary.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_binary.yaml rename to backends/vulkan/runtime/graph/ops/glsl/binary/q8ta/q8ta_binary.yaml diff --git a/backends/vulkan/runtime/graph/ops/impl/BinaryOp.cpp b/backends/vulkan/runtime/graph/ops/impl/binary/BinaryOp.cpp similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/BinaryOp.cpp rename to backends/vulkan/runtime/graph/ops/impl/binary/BinaryOp.cpp diff --git a/backends/vulkan/runtime/graph/ops/impl/BinaryScalarOp.cpp b/backends/vulkan/runtime/graph/ops/impl/binary/BinaryScalarOp.cpp similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/BinaryScalarOp.cpp rename to backends/vulkan/runtime/graph/ops/impl/binary/BinaryScalarOp.cpp diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taBinary.cpp b/backends/vulkan/runtime/graph/ops/impl/binary/q8ta/Q8taBinary.cpp similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Q8taBinary.cpp rename to backends/vulkan/runtime/graph/ops/impl/binary/q8ta/Q8taBinary.cpp diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taBinary.h b/backends/vulkan/runtime/graph/ops/impl/binary/q8ta/Q8taBinary.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Q8taBinary.h rename to backends/vulkan/runtime/graph/ops/impl/binary/q8ta/Q8taBinary.h diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taBinary.cpp b/backends/vulkan/test/custom_ops/impl/TestQ8taBinary.cpp index e3c3e6e2642..00b042c8180 100644 --- a/backends/vulkan/test/custom_ops/impl/TestQ8taBinary.cpp +++ b/backends/vulkan/test/custom_ops/impl/TestQ8taBinary.cpp @@ -9,8 +9,8 @@ #include #include -#include #include +#include namespace vkcompute { diff --git a/backends/vulkan/test/test_vulkan_shader_names.py b/backends/vulkan/test/test_vulkan_shader_names.py index 1535e6c9653..d7843a075e2 100644 --- a/backends/vulkan/test/test_vulkan_shader_names.py +++ b/backends/vulkan/test/test_vulkan_shader_names.py @@ -24,6 +24,7 @@ """ import importlib.util +import os import re import unittest from pathlib import Path @@ -42,7 +43,12 @@ _spec.loader.exec_module(gen_vulkan_spv) # The yaml templates whose variants `add_binary_op_node` dispatches into. -_BINARY_TEMPLATES = ("binary_op_buffer", "binary_op_texture") +_BINARY_BUFFER_TEMPLATE = os.path.join("binary", "binary_op_buffer") +_BINARY_TEXTURE_TEMPLATE = os.path.join("binary", "binary_op_texture") +_BINARY_TEMPLATES = ( + _BINARY_BUFFER_TEMPLATE, + _BINARY_TEXTURE_TEMPLATE, +) def _function_body(text: str, name: str) -> str: @@ -113,8 +119,8 @@ def test_int32_eq_shaders_are_named_for_the_dispatcher(self) -> None: build. Kokoro's synthesizer aborted on it. """ for template, expected in ( - ("binary_op_buffer", "binary_eq_buffer_int32"), - ("binary_op_texture", "binary_eq_texture3d_int32"), + (_BINARY_BUFFER_TEMPLATE, "binary_eq_buffer_int32"), + (_BINARY_TEXTURE_TEMPLATE, "binary_eq_texture3d_int32"), ): with self.subTest(template=template): self.assertIn(expected, self.names[template]) From d312aeac2cffc07fae6ae029e86732b056a3d2b6 Mon Sep 17 00:00:00 2001 From: Stephen Jia Date: Mon, 12 Oct 2026 12:45:47 -0700 Subject: [PATCH 4/7] [ET-VK][convolution] Nest convolution sources by route Pull Request resolved: https://github.com/pytorch/executorch/pull/23492 ## Ulterior Motive Organize Vulkan operators by implementation family so operators with multiple routes can use clear, scalable directory layouts. ## Rationale **What**: Move convolution C++ implementations, GLSL templates, YAML variant definitions, and convolution-only shader headers under `convolution/` in both `impl/` and `glsl/`, split into `conv1d/` and `conv2d/`. Split the monolithic `Convolution.cpp` and `Q8taConv2d.{h,cpp}` so each file holds either routing logic or a single implementation route. **Why**: Convolution is implemented through several routes (direct, depthwise, pointwise, im2col/GEMM, transposed), and the q8ta quantized family repeats most of them. In the flat directories their sources were interleaved by name, and single files mixed routing with several route implementations. This prepares for adding further routes such as Winograd. ## Details - Layout (`impl/` and `glsl/` mirror each other) - `convolution/`: `Convolution.{h,cpp}` registers `aten.convolution` and dispatches to conv1d or conv2d by input rank; it also holds `prepack_biases` and `check_conv_args`, used by both. - `convolution/conv1d/`: `Conv1d.{h,cpp}` (`conv1d_impl` routing), `Conv1dDirect.{h,cpp}`, `Conv1dDW.cpp`, `Conv1dPW.cpp`, and the conv1d shaders. - `convolution/conv2d/`: `Conv2d.{h,cpp}` (`conv2d_impl` routing, `Conv2dMethod`, `resize_conv2d_node`), `Conv2dDirect.{h,cpp}` (direct and transposed shader dispatch), `Conv2dDW.cpp`, `Conv2dPW.cpp`, `ConvolutionUtils.h`, `conv2d_common.glslh`, and the direct/depthwise/pointwise shaders. - `convolution/conv2d/{im2col,transposed}/`: im2col/GEMM sources and shaders; transposed shaders. - `convolution/conv2d/q8ta/`: `Q8taConv2d.{h,cpp}` (`q8ta_conv2d` routing and the im2col heuristic, formerly `Q8taConv2dRoute`), `Q8taConv2dCommon.{h,cpp}` (`ActivationType`, packed-dim checks), `Q8taConv2dDirect.{h,cpp}` (general route), `Q8taConv2dDW.{h,cpp}`, `Q8taConv2dPW.{h,cpp}`, plus `im2col/` and `transposed/`. Each route has its own header; consumers include only what they use. - `convolution/conv2d/quantized_legacy/`: `QuantizedConvolution.{h,cpp}` and its shaders and headers. - Weight packing shaders sit in `pack_weights/` subdirectories of the route that dispatches them (`conv2d/`, `conv2d/im2col/`, `conv2d/q8ta/`, `conv2d/transposed/`). - Naming - Functions that add a `DynamicDispatchNode` are named `add_*_node()`; functions that call `add_*_node()` to implement an operator are named `*_impl()`. Registered operator names are unchanged. - `add_conv2d_node` is folded into `conv2d_impl`. Its test-only `force_direct` flag is removed; the conv2d test's `direct` selector calls `conv2d_direct_impl` directly, which is equivalent for its sliding-window cases. - Includes - Route shaders include `conv2d_common.glslh` unchanged through ancestor include resolution. - Root shaders `quantize_and_pack_4w4c` and `unpack_4w4c_and_dequantize` include `convolution/conv2d/conv2d_common.glslh`. - Headers shared with other operator families (`linear_*`, `conv2d_fp_input_tile_load.glslh`, indexing headers) stay at the shader root. - Moved files are recorded as renames; files split from or merged into another file are recorded as copies of their source, so reviews show moved code as unchanged. - Moving code into new files exposes it to CLANGTIDY, so its findings there are fixed: `utils::safe_downcast` for narrowing, `(void)` for unused parameters, an anonymous namespace for the file-local `Conv2dParams`, and removal of an unused `ShaderNameUtils.h` include. Generated shader names and runtime dispatch behavior are unchanged. ghstack-source-id: 446951717 @exported-using-ghexport Differential Revision: [D123450342](https://our.internmc.facebook.com/intern/diff/D123450342/) --- backends/vulkan/op_registry.py | 4 +- .../glsl/{ => convolution/conv1d}/conv1d.glsl | 0 .../glsl/{ => convolution/conv1d}/conv1d.yaml | 0 .../{ => convolution/conv1d}/conv1d_dw.glsl | 0 .../{ => convolution/conv1d}/conv1d_dw.yaml | 0 .../{ => convolution/conv1d}/conv1d_pw.glsl | 0 .../{ => convolution/conv1d}/conv1d_pw.yaml | 0 .../glsl/{ => convolution/conv2d}/conv2d.glsl | 0 .../glsl/{ => convolution/conv2d}/conv2d.yaml | 0 .../conv2d}/conv2d_common.glslh | 0 .../{ => convolution/conv2d}/conv2d_dw.glsl | 0 .../{ => convolution/conv2d}/conv2d_dw.yaml | 0 .../conv2d}/conv2d_dw_output_tile.glsl | 0 .../conv2d}/conv2d_dw_output_tile.yaml | 0 .../conv2d}/conv2d_dw_sned_output_tile.glsl | 0 .../conv2d}/conv2d_dw_sned_output_tile.yaml | 0 .../{ => convolution/conv2d}/conv2d_pw.glsl | 0 .../{ => convolution/conv2d}/conv2d_pw.yaml | 0 .../conv2d}/conv2d_pw_s1p0.glsl | 0 .../conv2d}/conv2d_pw_s1p0.yaml | 0 .../conv2d}/conv2d_pw_tiled.glsl | 0 .../conv2d}/conv2d_pw_tiled.yaml | 0 .../conv2d/im2col}/conv2d_gemm.glsl | 0 .../conv2d/im2col}/conv2d_gemm.yaml | 0 .../conv2d/im2col}/conv2d_im2col.glsl | 0 .../conv2d/im2col}/conv2d_im2col.yaml | 0 .../pack_conv2d_gemm_weight.glsl | 0 .../pack_conv2d_gemm_weight.yaml | 0 .../conv2d_dw_prepack_weights.glsl | 0 .../conv2d_dw_prepack_weights.yaml | 0 .../pack_weights}/conv2d_prepack_weights.glsl | 0 .../pack_weights}/conv2d_prepack_weights.yaml | 0 .../conv2d/q8ta}/conv2d_dw_q8_utils.glslh | 0 .../q8ta}/conv2d_dw_q8ta_q8csw_q8to.glsl | 0 .../q8ta}/conv2d_dw_q8ta_q8csw_q8to.yaml | 0 .../conv2d/q8ta/im2col}/q8ta_im2col.glsl | 0 .../conv2d/q8ta/im2col}/q8ta_im2col.yaml | 0 .../pack_q8_conv2d_dw_weights.glsl | 0 .../pack_q8_conv2d_dw_weights.yaml | 0 .../pack_weights}/pack_q8_conv2d_weights.glsl | 0 .../pack_weights}/pack_q8_conv2d_weights.yaml | 0 .../conv2d/q8ta}/q8ta_conv2d.glsl | 0 .../conv2d/q8ta}/q8ta_conv2d.yaml | 0 .../conv2d/q8ta}/q8ta_conv2d_dw.glsl | 0 .../conv2d/q8ta}/q8ta_conv2d_dw.yaml | 0 .../conv2d/q8ta}/q8ta_conv2d_pw.glsl | 0 .../conv2d/q8ta}/q8ta_conv2d_pw.yaml | 0 .../transposed}/q8ta_conv2d_transposed.glsl | 0 .../transposed}/q8ta_conv2d_transposed.yaml | 0 .../conv2d/quantized_legacy}/col2im.glsl | 0 .../conv2d/quantized_legacy}/col2im.yaml | 0 .../conv2d_fp_im2col_block.glslh | 0 .../conv2d_fp_im2col_block_load.glslh | 0 .../conv2d_fp_im2col_block_store.glslh | 0 .../conv2d_int8_input_block_load.glslh | 0 .../conv2d_int8_input_tile_load.glslh | 0 .../conv2d_int8_output_tile_store.glslh | 0 .../conv2d_pw_q8ta_q8csw_q8to_tiled.glsl | 0 .../conv2d_pw_q8ta_q8csw_q8to_tiled.yaml | 0 .../quantized_legacy}/conv2d_q8_utils.glslh | 0 .../conv2d_q8csw_linear_tiled.glsl | 0 .../conv2d_q8csw_linear_tiled.yaml | 0 .../conv2d_q8ta_q8csw_linear_tiled.glsl | 0 .../conv2d_q8ta_q8csw_linear_tiled.yaml | 0 .../conv2d_q8ta_q8csw_q8to.glsl | 0 .../conv2d_q8ta_q8csw_q8to.yaml | 0 .../conv2d_q8ta_q8csw_q8to_linear_tiled.glsl | 0 .../conv2d_q8ta_q8csw_q8to_linear_tiled.yaml | 0 .../conv2d/quantized_legacy}/im2col.glsl | 0 .../conv2d/quantized_legacy}/im2col.yaml | 0 .../quantized_legacy}/im2col_packed_int8.glsl | 0 .../quantized_legacy}/im2col_packed_int8.yaml | 0 .../im2col_packed_int8_utils.glslh | 0 .../quantize_and_pack_im2col.glsl | 0 .../quantize_and_pack_im2col.yaml | 0 .../conv2d/transposed}/conv_transpose2d.glsl | 0 .../conv2d/transposed}/conv_transpose2d.yaml | 0 .../conv_transpose2d_prepack_weights.glsl | 0 .../conv_transpose2d_prepack_weights.yaml | 0 .../ops/glsl/quantize_and_pack_4w4c.glsl | 2 +- .../ops/glsl/unpack_4w4c_and_dequantize.glsl | 2 +- .../runtime/graph/ops/impl/Convolution.cpp | 871 ------------------ .../runtime/graph/ops/impl/Q8taConv2d.h | 216 ----- .../runtime/graph/ops/impl/Q8taLinear.cpp | 2 +- .../runtime/graph/ops/impl/Q8taLinearGemv.cpp | 2 +- .../ops/impl/convolution/Convolution.cpp | 84 ++ .../graph/ops/impl/convolution/Convolution.h | 28 + .../ops/impl/convolution/conv1d/Conv1d.cpp | 100 ++ .../ops/impl/convolution/conv1d/Conv1d.h | 17 + .../{ => convolution/conv1d}/Conv1dDW.cpp | 4 +- .../impl/convolution/conv1d/Conv1dDirect.cpp | 179 ++++ .../impl/convolution/conv1d/Conv1dDirect.h | 29 + .../{ => convolution/conv1d}/Conv1dPW.cpp | 4 +- .../ops/impl/convolution/conv2d/Conv2d.cpp | 234 +++++ .../conv2d/Conv2d.h} | 40 +- .../{ => convolution/conv2d}/Conv2dDW.cpp | 3 +- .../impl/convolution/conv2d/Conv2dDirect.cpp | 402 ++++++++ .../impl/convolution/conv2d/Conv2dDirect.h | 33 + .../{ => convolution/conv2d}/Conv2dPW.cpp | 3 +- .../conv2d}/ConvolutionUtils.h | 0 .../conv2d/im2col}/Conv2dGemm.cpp | 6 +- .../conv2d/im2col}/Conv2dGemm.h | 0 .../conv2d/im2col}/Conv2dIm2Col.cpp | 2 +- .../conv2d/im2col}/Conv2dIm2Col.h | 0 .../conv2d/q8ta/Q8taConv2d.cpp} | 59 +- .../conv2d/q8ta/Q8taConv2d.h} | 4 + .../conv2d/q8ta/Q8taConv2dCommon.cpp | 37 + .../conv2d/q8ta/Q8taConv2dCommon.h | 28 + .../conv2d/q8ta}/Q8taConv2dDW.cpp | 12 +- .../convolution/conv2d/q8ta/Q8taConv2dDW.h | 61 ++ .../conv2d/q8ta/Q8taConv2dDirect.cpp} | 85 +- .../conv2d/q8ta/Q8taConv2dDirect.h | 50 + .../conv2d/q8ta}/Q8taConv2dPW.cpp | 5 +- .../convolution/conv2d/q8ta/Q8taConv2dPW.h | 53 ++ .../conv2d/q8ta/im2col}/Q8taConv2dIm2Col.cpp | 11 +- .../conv2d/q8ta/im2col/Q8taConv2dIm2Col.h | 71 ++ .../q8ta/transposed}/Q8taConv2dTransposed.cpp | 12 +- .../q8ta/transposed/Q8taConv2dTransposed.h | 19 + .../QuantizedConvolution.cpp | 13 +- .../quantized_legacy}/QuantizedConvolution.h | 2 +- .../test/custom_ops/impl/TestConv2d.cpp | 20 +- .../test/custom_ops/impl/TestConv2dDw.cpp | 3 +- .../test/custom_ops/impl/TestQ8taConv2d.cpp | 3 +- .../custom_ops/q8ta_conv2d_route_test.cpp | 2 +- .../q8ta_conv2d_stream_plan_test.cpp | 2 +- .../vulkan/test/custom_ops/test_conv2d.cpp | 2 +- .../test/custom_ops/test_q8ta_conv2d.cpp | 3 +- .../test/custom_ops/test_q8ta_conv2d_pw.cpp | 2 +- .../op_tests/conv2d_gemm_dynamic_test.cpp | 2 +- 129 files changed, 1574 insertions(+), 1254 deletions(-) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv1d}/conv1d.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv1d}/conv1d.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv1d}/conv1d_dw.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv1d}/conv1d_dw.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv1d}/conv1d_pw.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv1d}/conv1d_pw.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d_common.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d_dw.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d_dw.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d_dw_output_tile.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d_dw_output_tile.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d_dw_sned_output_tile.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d_dw_sned_output_tile.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d_pw.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d_pw.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d_pw_s1p0.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d_pw_s1p0.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d_pw_tiled.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d}/conv2d_pw_tiled.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/im2col}/conv2d_gemm.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/im2col}/conv2d_gemm.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/im2col}/conv2d_im2col.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/im2col}/conv2d_im2col.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/im2col/pack_weights}/pack_conv2d_gemm_weight.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/im2col/pack_weights}/pack_conv2d_gemm_weight.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/pack_weights}/conv2d_dw_prepack_weights.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/pack_weights}/conv2d_dw_prepack_weights.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/pack_weights}/conv2d_prepack_weights.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/pack_weights}/conv2d_prepack_weights.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta}/conv2d_dw_q8_utils.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta}/conv2d_dw_q8ta_q8csw_q8to.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta}/conv2d_dw_q8ta_q8csw_q8to.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta/im2col}/q8ta_im2col.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta/im2col}/q8ta_im2col.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta/pack_weights}/pack_q8_conv2d_dw_weights.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta/pack_weights}/pack_q8_conv2d_dw_weights.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta/pack_weights}/pack_q8_conv2d_weights.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta/pack_weights}/pack_q8_conv2d_weights.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta}/q8ta_conv2d.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta}/q8ta_conv2d.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta}/q8ta_conv2d_dw.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta}/q8ta_conv2d_dw.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta}/q8ta_conv2d_pw.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta}/q8ta_conv2d_pw.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta/transposed}/q8ta_conv2d_transposed.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/q8ta/transposed}/q8ta_conv2d_transposed.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/col2im.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/col2im.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_fp_im2col_block.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_fp_im2col_block_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_fp_im2col_block_store.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_int8_input_block_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_int8_input_tile_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_int8_output_tile_store.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_pw_q8ta_q8csw_q8to_tiled.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_q8_utils.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_q8csw_linear_tiled.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_q8csw_linear_tiled.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_q8ta_q8csw_linear_tiled.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_q8ta_q8csw_linear_tiled.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_q8ta_q8csw_q8to.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_q8ta_q8csw_q8to.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/conv2d_q8ta_q8csw_q8to_linear_tiled.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/im2col.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/im2col.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/im2col_packed_int8.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/im2col_packed_int8.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/im2col_packed_int8_utils.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/quantize_and_pack_im2col.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/quantized_legacy}/quantize_and_pack_im2col.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/transposed}/conv_transpose2d.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/transposed}/conv_transpose2d.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/transposed/pack_weights}/conv_transpose2d_prepack_weights.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => convolution/conv2d/transposed/pack_weights}/conv_transpose2d_prepack_weights.yaml (100%) delete mode 100644 backends/vulkan/runtime/graph/ops/impl/Convolution.cpp delete mode 100644 backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.h create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.cpp create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.h create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.cpp create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.h rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv1d}/Conv1dDW.cpp (97%) create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.cpp create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.h rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv1d}/Conv1dPW.cpp (98%) create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2d.cpp rename backends/vulkan/runtime/graph/ops/impl/{Convolution.h => convolution/conv2d/Conv2d.h} (50%) rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv2d}/Conv2dDW.cpp (98%) create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.cpp create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.h rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv2d}/Conv2dPW.cpp (97%) rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv2d}/ConvolutionUtils.h (100%) rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv2d/im2col}/Conv2dGemm.cpp (98%) rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv2d/im2col}/Conv2dGemm.h (100%) rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv2d/im2col}/Conv2dIm2Col.cpp (98%) rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv2d/im2col}/Conv2dIm2Col.h (100%) rename backends/vulkan/runtime/graph/ops/impl/{Q8taConv2dRoute.cpp => convolution/conv2d/q8ta/Q8taConv2d.cpp} (70%) rename backends/vulkan/runtime/graph/ops/impl/{Q8taConv2dRoute.h => convolution/conv2d/q8ta/Q8taConv2d.h} (84%) create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.cpp create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.h rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv2d/q8ta}/Q8taConv2dDW.cpp (97%) create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDW.h rename backends/vulkan/runtime/graph/ops/impl/{Q8taConv2d.cpp => convolution/conv2d/q8ta/Q8taConv2dDirect.cpp} (84%) create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDirect.h rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv2d/q8ta}/Q8taConv2dPW.cpp (98%) create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dPW.h rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv2d/q8ta/im2col}/Q8taConv2dIm2Col.cpp (96%) create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.h rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv2d/q8ta/transposed}/Q8taConv2dTransposed.cpp (95%) create mode 100644 backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/transposed/Q8taConv2dTransposed.h rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv2d/quantized_legacy}/QuantizedConvolution.cpp (98%) rename backends/vulkan/runtime/graph/ops/impl/{ => convolution/conv2d/quantized_legacy}/QuantizedConvolution.h (78%) diff --git a/backends/vulkan/op_registry.py b/backends/vulkan/op_registry.py index 461aeeaafcc..22c9da4aab1 100644 --- a/backends/vulkan/op_registry.py +++ b/backends/vulkan/op_registry.py @@ -896,7 +896,7 @@ def register_max_pool2d_with_indices(): # ============================================================================= -# Convolution.cpp +# convolution/Convolution.cpp # ============================================================================= @@ -989,7 +989,7 @@ def pick_conv_storage( # ============================================================================= -# Q8taConv2d*.cpp +# convolution/conv2d/q8ta/**/Q8taConv2d*.cpp # ============================================================================= diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv1d.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv1d.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv1d.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv1d.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv1d_dw.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_dw.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv1d_dw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_dw.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv1d_dw.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_dw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv1d_dw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_dw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv1d_pw.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_pw.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv1d_pw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_pw.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv1d_pw.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_pw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv1d_pw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_pw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_common.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_common.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_common.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_common.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_output_tile.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_output_tile.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_output_tile.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_output_tile.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_output_tile.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_output_tile.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_output_tile.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_output_tile.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_sned_output_tile.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_sned_output_tile.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_sned_output_tile.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_sned_output_tile.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_sned_output_tile.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_sned_output_tile.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_sned_output_tile.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_sned_output_tile.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_s1p0.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_s1p0.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_s1p0.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_s1p0.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_s1p0.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_s1p0.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_s1p0.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_s1p0.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_tiled.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_tiled.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_gemm.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_gemm.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_gemm.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_gemm.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_gemm.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_gemm.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_gemm.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_gemm.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_im2col.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_im2col.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_im2col.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_im2col.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_im2col.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_im2col.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_im2col.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_im2col.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_conv2d_gemm_weight.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/pack_weights/pack_conv2d_gemm_weight.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_conv2d_gemm_weight.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/pack_weights/pack_conv2d_gemm_weight.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_conv2d_gemm_weight.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/pack_weights/pack_conv2d_gemm_weight.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_conv2d_gemm_weight.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/pack_weights/pack_conv2d_gemm_weight.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_prepack_weights.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_dw_prepack_weights.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_prepack_weights.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_dw_prepack_weights.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_prepack_weights.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_dw_prepack_weights.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_prepack_weights.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_dw_prepack_weights.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_prepack_weights.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_prepack_weights.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_prepack_weights.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_prepack_weights.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_prepack_weights.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_prepack_weights.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_prepack_weights.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_prepack_weights.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_q8_utils.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/conv2d_dw_q8_utils.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_q8_utils.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/conv2d_dw_q8_utils.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_q8ta_q8csw_q8to.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/conv2d_dw_q8ta_q8csw_q8to.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_q8ta_q8csw_q8to.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/conv2d_dw_q8ta_q8csw_q8to.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_q8ta_q8csw_q8to.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/conv2d_dw_q8ta_q8csw_q8to.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_q8ta_q8csw_q8to.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/conv2d_dw_q8ta_q8csw_q8to.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_im2col.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/im2col/q8ta_im2col.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_im2col.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/im2col/q8ta_im2col.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_im2col.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/im2col/q8ta_im2col.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_im2col.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/im2col/q8ta_im2col.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_dw_weights.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_dw_weights.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_dw_weights.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_dw_weights.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_dw_weights.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_dw_weights.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_dw_weights.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_dw_weights.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_weights.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_weights.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_weights.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_weights.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_weights.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_weights.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_weights.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_weights.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_dw.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_dw.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_dw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_dw.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_dw.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_dw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_dw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_dw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_pw.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_pw.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_pw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_pw.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_pw.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_pw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_pw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_pw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_transposed.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/transposed/q8ta_conv2d_transposed.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_transposed.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/transposed/q8ta_conv2d_transposed.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_transposed.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/transposed/q8ta_conv2d_transposed.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_transposed.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/transposed/q8ta_conv2d_transposed.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/col2im.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/col2im.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/col2im.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/col2im.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/col2im.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/col2im.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/col2im.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/col2im.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_store.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block_store.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_store.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_input_block_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_input_block_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_input_block_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_input_block_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_input_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_input_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_input_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_input_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_output_tile_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_output_tile_store.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_output_tile_store.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_output_tile_store.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_q8ta_q8csw_q8to_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_pw_q8ta_q8csw_q8to_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_q8ta_q8csw_q8to_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_pw_q8ta_q8csw_q8to_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8_utils.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8_utils.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8_utils.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8_utils.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8csw_linear_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8csw_linear_tiled.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8csw_linear_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8csw_linear_tiled.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8csw_linear_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8csw_linear_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8csw_linear_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8csw_linear_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_linear_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_linear_tiled.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_linear_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_linear_tiled.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_linear_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_linear_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_linear_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_linear_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to_linear_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to_linear_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to_linear_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to_linear_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/im2col.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/im2col.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/im2col.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/im2col.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/im2col_packed_int8.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col_packed_int8.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/im2col_packed_int8.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col_packed_int8.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/im2col_packed_int8.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col_packed_int8.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/im2col_packed_int8.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col_packed_int8.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/im2col_packed_int8_utils.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col_packed_int8_utils.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/im2col_packed_int8_utils.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col_packed_int8_utils.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_im2col.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/quantize_and_pack_im2col.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_im2col.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/quantize_and_pack_im2col.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_im2col.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/quantize_and_pack_im2col.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_im2col.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/quantize_and_pack_im2col.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/conv_transpose2d.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/conv_transpose2d.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/conv_transpose2d.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/conv_transpose2d.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d_prepack_weights.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/pack_weights/conv_transpose2d_prepack_weights.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d_prepack_weights.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/pack_weights/conv_transpose2d_prepack_weights.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d_prepack_weights.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/pack_weights/conv_transpose2d_prepack_weights.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d_prepack_weights.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/pack_weights/conv_transpose2d_prepack_weights.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4w4c.glsl b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4w4c.glsl index 205929d7456..6f58021be7f 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4w4c.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4w4c.glsl @@ -28,7 +28,7 @@ $if INPUT_STORAGE == "buffer": layout(std430) buffer; -#include "conv2d_common.glslh" +#include "convolution/conv2d/conv2d_common.glslh" ${layout_declare_tensor(B, "w", "t_packed_int8_input", "int", OUTPUT_STORAGE, is_scalar_array=False)} ${layout_declare_tensor(B, "r", "t_fp_input", DTYPE, INPUT_STORAGE)} diff --git a/backends/vulkan/runtime/graph/ops/glsl/unpack_4w4c_and_dequantize.glsl b/backends/vulkan/runtime/graph/ops/glsl/unpack_4w4c_and_dequantize.glsl index 3baa21f54bf..fd7fa0aa7f1 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/unpack_4w4c_and_dequantize.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/unpack_4w4c_and_dequantize.glsl @@ -28,7 +28,7 @@ $if INPUT_STORAGE == "buffer": layout(std430) buffer; -#include "conv2d_common.glslh" +#include "convolution/conv2d/conv2d_common.glslh" ${layout_declare_tensor(B, "w", "t_fp_output", DTYPE, OUTPUT_STORAGE)} ${layout_declare_tensor(B, "r", "t_packed_int8_output", "int", INPUT_STORAGE, is_scalar_array=False)} diff --git a/backends/vulkan/runtime/graph/ops/impl/Convolution.cpp b/backends/vulkan/runtime/graph/ops/impl/Convolution.cpp deleted file mode 100644 index 272e886b7b1..00000000000 --- a/backends/vulkan/runtime/graph/ops/impl/Convolution.cpp +++ /dev/null @@ -1,871 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include - -#include - -#include -#include -#include - -#include - -#include -#include - -#include - -namespace vkcompute { - -enum class Conv2dMethod : uint8_t { - Depthwise, - Pointwise, - SlidingWindow, - Transposed, -}; - -void resize_conv2d_node( - ComputeGraph* graph, - const std::vector& args, - const std::vector& extra_args) { - const ValueRef out = args.at(0).refs.at(0); - const ValueRef self = args.at(1).refs.at(0); - - size_t ndim = graph->dim_of(self); - std::vector new_out_sizes(ndim); - const bool transposed = graph->get_bool(extra_args.at(4)); - - std::vector self_sizes = graph->sizes_of(self); - // Batch, Channel - if (ndim == 4) { - new_out_sizes.at(ndim - 4) = self_sizes.at(ndim - 4); - } - - TensorRefPtr weight_ref = graph->get_tref(extra_args.at(0)); - const auto& weight_sizes = weight_ref->sizes; - new_out_sizes.at(ndim - 3) = - transposed ? weight_sizes.at(ndim - 3) : weight_sizes.at(ndim - 4); - - // Height, Width - const auto& new_out_sizes_hw = calc_out_sizes_hw( - *graph, - self_sizes, - extra_args.at(0), - /*kernel_size_only = */ false, - {extra_args.at(1), extra_args.at(2), extra_args.at(3), extra_args.at(5)}, - transposed); - new_out_sizes.at(ndim - 2) = new_out_sizes_hw.at(0); - new_out_sizes.at(ndim - 1) = new_out_sizes_hw.at(1); - - graph->virtual_resize(out, new_out_sizes); -} - -void resize_conv1d_node( - ComputeGraph* graph, - const std::vector& args, - const std::vector& extra_args) { - const ValueRef out = args.at(0).refs.at(0); - const ValueRef self = args.at(1).refs.at(0); - TensorRefPtr weight_ref = graph->get_tref(extra_args.at(0)); - - const int64_t stride_size = graph->get_int_list(extra_args.at(1))->at(0); - const int64_t padding_size = graph->get_int_list(extra_args.at(2))->at(0); - const int64_t dilation_size = graph->get_int_list(extra_args.at(3))->at(0); - - const std::vector& weight_sizes = weight_ref->sizes; - - const std::vector in_sizes = graph->sizes_of(self); - const size_t ndim = in_sizes.size(); - std::vector new_out_sizes(ndim); - - const int64_t kernel_size = weight_sizes.at(2); - const int64_t in_length = in_sizes.at(2); - - new_out_sizes.at(0) = in_sizes.at(0); - new_out_sizes.at(1) = weight_sizes.at(0); - new_out_sizes.at(2) = calc_out_size( - in_length, kernel_size, stride_size, padding_size, dilation_size, false); - - graph->virtual_resize(out, new_out_sizes); -} - -ValueRef prepack_biases( - ComputeGraph& graph, - const ValueRef vref, - const ValueRef weight, - const bool transposed, - const utils::StorageType storage_type, - const utils::GPUMemoryLayout memory_layout) { - auto sizes = graph.sizes_of(weight); - const int64_t out_channels = transposed ? sizes.at(1) : sizes.at(0); - - ValueRef v = graph.add_tensor( - {out_channels}, graph.dtype_of(weight), storage_type, memory_layout); - - vkapi::ShaderInfo shader = - get_nchw_to_tensor_shader(graph, v, graph.get_staging_dtype_for(weight)); - - vkapi::ParamsBindList param_buffers = {}; - if (graph.is_buffer_storage(v)) { - param_buffers.append(graph.buffer_meta_ubo(v)); - } else { - param_buffers.append(graph.texture_meta_ubo(v)); - } - - std::vector pcs; - if (graph.is_buffer_storage(v)) { - pcs = {graph.sizes_pc_of(v), graph.strides_pc_of(v), graph.numel_pc_of(v)}; - } - - graph.prepack_nodes().emplace_back(new PrepackNode( - graph, - shader, - graph.create_gwg(v), - graph.create_lwg(v), - vref, - v, - param_buffers, - // Specialization constants - {graph.hashed_layout_of(v)}, - pcs)); - - return v; -} - -vkapi::ShaderInfo get_conv2d_shader( - ComputeGraph& graph, - const ValueRef out, - const bool prepack_weights, - const Conv2dMethod method, - const ValueRef weight, - const bool clamp_out = false, - const bool stride_equals_dilation = false, - const bool stride_1_padding_0 = false) { - std::string kernel_name; - kernel_name.reserve(kShaderNameReserve); - switch (method) { - case Conv2dMethod::Depthwise: - kernel_name = "conv2d_dw"; - break; - case Conv2dMethod::Pointwise: - if (prepack_weights) { - kernel_name = "conv2d"; - } else { - kernel_name = stride_1_padding_0 ? "conv2d_pw_s1p0" : "conv2d_pw"; - } - break; - case Conv2dMethod::SlidingWindow: - kernel_name = "conv2d"; - break; - case Conv2dMethod::Transposed: - kernel_name = "conv_transpose2d"; - break; - } - if (prepack_weights) { - kernel_name += "_prepack_weights"; - } else if (clamp_out) { - kernel_name += "_clamp"; - } - add_dtype_suffix(kernel_name, graph.dtype_of(out)); - - if (prepack_weights) { - add_dtype_suffix(kernel_name, graph.get_staging_dtype_for(weight)); - } - - return VK_KERNEL_FROM_STR(kernel_name); -} - -std::vector get_final_sizes( - const std::vector& original_sizes, - const Conv2dMethod method) { - int64_t batch_padded = utils::align_up_4(utils::val_at(-4, original_sizes)); - int64_t channels_padded = - utils::align_up_4(utils::val_at(-3, original_sizes)); - int64_t height = utils::val_at(-2, original_sizes); - int64_t width = utils::val_at(-1, original_sizes); - - switch (method) { - case Conv2dMethod::Depthwise: - return std::vector{4, batch_padded / 4, height * width}; - case Conv2dMethod::Pointwise: - case Conv2dMethod::SlidingWindow: - return std::vector{ - 4, batch_padded * height / 4, channels_padded * width}; - case Conv2dMethod::Transposed: - return std::vector{ - 4, channels_padded * height / 4, batch_padded * width}; - } -} - -ValueRef prepack_weights( - ComputeGraph& graph, - const ValueRef vref, - const Conv2dMethod method) { - const auto original_sizes = graph.sizes_of(vref); - const auto final_sizes = get_final_sizes(original_sizes, method); - - ValueRef v = graph.add_tensor( - final_sizes, - graph.dtype_of(vref), - utils::kTexture2D, - utils::kChannelsPacked); - - vkapi::ShaderInfo shader = - get_conv2d_shader(graph, v, /*prepack_weights = */ true, method, vref); - - const auto original_sizes_pc = - utils::make_ivec4(original_sizes, /*reverse = */ true); - graph.prepack_nodes().emplace_back(new PrepackNode( - graph, - shader, - graph.create_gwg(v), - graph.create_lwg(v), - vref, - v, - {}, - // Specialization constants - {graph.packed_dim_of(v)}, - {graph.sizes_pc_of(v), - PushConstantDataInfo(&original_sizes_pc, sizeof(original_sizes_pc))})); - - return v; -} - -void check_conv_args( - ComputeGraph& graph, - const ValueRef in, - const ValueRef out) { - VK_CHECK_COND(graph.packed_dim_of(in) == WHCN::kChannelsDim); - VK_CHECK_COND(graph.packed_dim_of(out) == WHCN::kChannelsDim); -} - -struct Conv2dParams final { - utils::ivec2 overlay_region; - int in_group_size; -}; - -struct OutputParams final { - float out_min; - float out_max; -}; - -Conv2dParams create_conv2d_params( - ComputeGraph& graph, - const ValueRef weight, - const Kernel2dParams& p, - const bool transposed) { - const auto& overlay_region = utils::make_ivec2({ - p.kernel_size[0] + (p.kernel_size[0] - 1) * (p.dilation[0] - 1), - p.kernel_size[1] + (p.kernel_size[1] - 1) * (p.dilation[1] - 1), - }); - const auto weight_sizes = graph.sizes_of(weight); - const int32_t in_group_size = utils::safe_downcast( - utils::align_up_4(transposed ? weight_sizes.at(0) : weight_sizes.at(1))); - return {overlay_region, in_group_size}; -} - -void check_conv2d_params(const Kernel2dParams& p, const bool transposed) { - if (transposed) { - if (p.dilation[0] > 1 || p.dilation[1] > 1) { - VK_THROW( - "aten.convolution.default: transposed = true, dilation > 1 is not supported yet!"); - } - } -} - -Conv2dMethod get_conv2d_method( - ComputeGraph& graph, - const ValueRef weight, - const int64_t groups, - const bool transposed) { - const auto weight_sizes = graph.sizes_of(weight); - if (!transposed && weight_sizes.at(0) == groups && weight_sizes.at(1) == 1) { - return Conv2dMethod::Depthwise; - } - if (transposed) { - return Conv2dMethod::Transposed; - } - if (weight_sizes.at(2) == 1 && weight_sizes.at(3) == 1) { - return Conv2dMethod::Pointwise; - } - return Conv2dMethod::SlidingWindow; -} - -// Decide whether a SlidingWindow conv2d should be computed via the -// im2col + GEMM path (conv2d_gemm_impl) instead of the direct convolution -// shader. Across 26 configs on Mali-G715 (buffer path) and Adreno SM8650 -// (texture path): FP32 cases were numerically verified against the reference; -// FP16 cases were routing/dispatch-validated only (the reference is float-only -// for the large shapes, so FP16 outputs were not numerically checked). -// -// Only called for SlidingWindow conv2d (1x1 is routed to conv2d_pw and -// Depthwise/Transposed are handled before the call site). -// -// Preconditions (fall back to direct conv if any fail — the im2col path is -// either not applicable or not beneficial): -// - groups == 1 -// - dilation == 1 (all dims) -// -// Selection rule: use im2col once the output channel count is large enough to -// amortize the fixed ~N*K_total im2col gather cost. -constexpr int64_t kIm2colMinCOut = 128; - -// A cheap gather is a second, independent reason to take im2col. The gather -// materializes an N x K_total matrix before the GEMM runs, and a large gather -// is fine when c_out is large because the GEMM reads it back c_out times. When -// c_out is small it is only worth paying if the matrix is small outright, which -// is where the direct shader on Mali loses badly: an 80x80 3x3 conv wants ~4M -// elements, a 640x640 9x9 conv wants ~100M. -constexpr int64_t kIm2colMaxCheapGatherElements = 32 * 1024 * 1024; - -bool should_use_conv2d_im2col( - ComputeGraph& graph, - const ValueRef weight_data, - const int64_t groups_val, - const Kernel2dParams& kernel_params, - const ValueRef out) { - if (groups_val != 1) { - return false; - } - if (kernel_params.dilation[0] != 1 || kernel_params.dilation[1] != 1) { - return false; - } - const auto weight_sizes = graph.sizes_of(weight_data); - const int64_t c_out = weight_sizes.at(0); - const int64_t k_total = - weight_sizes.at(1) * weight_sizes.at(2) * weight_sizes.at(3); - - const auto out_sizes = graph.sizes_of(out); - const size_t ndim = out_sizes.size(); - const int64_t n = out_sizes.at(ndim - 1) * out_sizes.at(ndim - 2); - return c_out >= kIm2colMinCOut || - (graph.device_is_mali() && n * k_total <= kIm2colMaxCheapGatherElements); -} - -GlobalWorkGrid create_conv2d_gwg( - ComputeGraph& graph, - const Conv2dMethod method, - const ValueRef out, - const ValueRef weight_data, - const bool stride_equals_dilation) { - if (method == Conv2dMethod::Pointwise) { - const utils::uvec3 image_extents = graph.logical_limits_of(out); - return GlobalWorkGrid( - {utils::div_up(image_extents[0u], 1u), - utils::div_up(image_extents[1u], 4u), - image_extents[2u]}, - kTiledWorkGrid); - } else { - return graph.create_gwg(out); - } -} - -// Determines which convolution method a dispatch uses. -// -// Depthwise and transposed convolutions have shader names of their own, but -// the name alone cannot separate pointwise from sliding window: the sliding -// window shader is itself named "conv2d", and a pointwise convolution also -// takes that name when its weights are prepacked. Those two are therefore -// separated by the weight's spatial extent. Shared by the global and local -// workgroup size functions below so that the two cannot disagree about the -// same dispatch. -Conv2dMethod infer_conv2d_method_from_shader( - ComputeGraph* graph, - const vkapi::ShaderInfo& shader, - const ValueRef weight_data) { - const std::string& kernel_name = shader.kernel_name; - // Checked before the plain "conv2d" test below, which "conv2d_dw" and - // "conv2d_pw" would otherwise match too. - if (kernel_name.find("conv2d_dw") != std::string::npos) { - return Conv2dMethod::Depthwise; - } - if (kernel_name.find("conv2d_pw") != std::string::npos) { - return Conv2dMethod::Pointwise; - } - if (kernel_name.find("conv_transpose2d") != std::string::npos) { - return Conv2dMethod::Transposed; - } - const auto& weight_sizes = graph->get_tref(weight_data)->sizes; - if (weight_sizes.at(2) == 1 && weight_sizes.at(3) == 1) { - return Conv2dMethod::Pointwise; - } - return Conv2dMethod::SlidingWindow; -} - -// Custom global workgroup size function for conv2d -GlobalWorkGrid conv2d_gwg( - ComputeGraph* graph, - const vkapi::ShaderInfo& shader, - const std::vector& args, - const std::vector& resize_args) { - const ValueRef out = args.at(0).refs.at(0); - const ValueRef weight_data = resize_args.at(0); - - const Conv2dMethod method = - infer_conv2d_method_from_shader(graph, shader, weight_data); - - // Determine stride_equals_dilation from shader name - bool stride_equals_dilation = - shader.kernel_name.find("_sned") == std::string::npos; - - const GlobalWorkGrid wg_size = create_conv2d_gwg( - *graph, method, out, weight_data, stride_equals_dilation); - - if (method == Conv2dMethod::Pointwise) { - utils::uvec3 pointwise_wg_size = {wg_size[0] * wg_size[1], wg_size[2], 1u}; - - if (shader.kernel_name.find("s1p0") != std::string::npos) { - pointwise_wg_size[0] *= 4; - } - return GlobalWorkGrid(pointwise_wg_size, kTiledWorkGrid); - } - - return wg_size; -} - -// Custom local workgroup size function for conv2d -LocalWorkGroup conv2d_lwg( - ComputeGraph* graph, - const vkapi::ShaderInfo& shader, - const GlobalWorkGrid& gwg, - const std::vector& args, - const std::vector& resize_args) { - (void)args; - - const ValueRef weight_data = resize_args.at(0); - const Conv2dMethod method = - infer_conv2d_method_from_shader(graph, shader, weight_data); - - if (method == Conv2dMethod::Pointwise) { - uint32_t lwg_y = 1; - if (gwg[1] % 8 == 0) { - lwg_y = 8; - } else if (gwg[1] % 4 == 0) { - lwg_y = 4; - } else if (gwg[1] % 2 == 0) { - lwg_y = 2; - } - return LocalWorkGroup(64u / lwg_y, lwg_y, 1u); - } else { - return graph->create_lwg(gwg); - } -} - -// Custom global workgroup size function for conv1d -GlobalWorkGrid conv1d_gwg( - ComputeGraph* graph, - const vkapi::ShaderInfo& shader, - const std::vector& args, - const std::vector& resize_args) { - (void)shader; - (void)resize_args; - const ValueRef out = args.at(0).refs.at(0); - - return GlobalWorkGrid( - {// out length - graph->size_at(-1, out), - // out channels - static_cast(graph->size_at(-2, out)), - // out batches - utils::div_up_4(graph->size_at(-3, out))}, - kTiledWorkGrid); -} - -void add_conv2d_node( - ComputeGraph& graph, - const ValueRef in, - const ValueRef weight_data, - const ValueRef bias, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef transposed, - const ValueRef output_padding, - const ValueRef groups, - const ValueRef out_min, - const ValueRef out_max, - const ValueRef out, - const bool clamp_out, - const bool force_direct) { - const bool transposed_val = graph.get_bool(transposed); - - float out_min_val = 0.0f; - float out_max_val = 0.0f; - if (out_min != kDummyValueRef) { - out_min_val = graph.extract_scalar(out_min); - } - if (out_max != kDummyValueRef) { - out_max_val = graph.extract_scalar(out_max); - } - - const int64_t groups_val = graph.get_int(groups); - - const Conv2dMethod method = - get_conv2d_method(graph, weight_data, groups_val, transposed_val); - - // Use tiled path for all pointwise conv2d - if (method == Conv2dMethod::Pointwise) { - return conv2d_pw_impl( - graph, - in, - weight_data, - bias, - stride, - padding, - out, - transposed_val, - clamp_out, - out_min_val, - out_max_val); - } - - if (method == Conv2dMethod::Depthwise) { - return conv2d_dw_impl( - graph, - in, - weight_data, - bias, - stride, - padding, - dilation, - out, - clamp_out, - out_min_val, - out_max_val); - } - - const Kernel2dParams kernel_params = create_kernel2d_params( - graph, - weight_data, - /*kernel_size_only = */ false, - stride, - padding, - dilation); - - // SlidingWindow conv2d: route to the im2col + GEMM path when the heuristic - // indicates it is beneficial, falling back to the direct convolution shader - // otherwise. `force_direct` bypasses the heuristic entirely and forces the - // direct path (used by tests to exercise the direct shader regardless of - // device); the default (false) reproduces the production routing exactly. - const bool use_im2col = !force_direct && - method == Conv2dMethod::SlidingWindow && - should_use_conv2d_im2col( - graph, weight_data, groups_val, kernel_params, out); - if (use_im2col) { - return conv2d_gemm_impl( - graph, - in, - weight_data, - bias, - stride, - padding, - dilation, - out, - clamp_out, - out_min_val, - out_max_val); - } - - ValueRef arg_weight = prepack_weights(graph, weight_data, method); - ValueRef arg_bias = prepack_biases( - graph, - bias, - weight_data, - transposed_val, - /* storage_type = */ utils::kTexture2D, - /* memory_layout = */ utils::kWidthPacked); - - const std::vector in_sizes = graph.sizes_of(in); - if (in_sizes.at(0) > 1) { - VK_THROW("conv2d: input batch size > 1 is not supported yet!"); - } - - check_conv_args(graph, in, out); - - Conv2dParams extra_params = - create_conv2d_params(graph, weight_data, kernel_params, transposed_val); - - const bool stride_equals_dilation = - (kernel_params.stride[0] == kernel_params.dilation[0] && - kernel_params.stride[1] == kernel_params.dilation[1]); - - const bool stride_1_padding_0 = - (kernel_params.stride[0] == 1 && kernel_params.stride[1] == 1 && - kernel_params.padding[0] == 0 && kernel_params.padding[1] == 0); - - OutputParams out_params = {out_min_val, out_max_val}; - - check_conv2d_params(kernel_params, transposed_val); - - vkapi::ShaderInfo shader = get_conv2d_shader( - graph, - out, - /*prepack_weights = */ false, - method, - weight_data, - clamp_out, - stride_equals_dilation, - stride_1_padding_0); - - vkapi::ParamsBindList param_buffers = { - graph.logical_limits_ubo(out), - graph.sizes_ubo(in), - graph.create_params_buffer(kernel_params), - graph.create_params_buffer(extra_params), - graph.create_params_buffer(out_params), - }; - - graph.execute_nodes().emplace_back(new DynamicDispatchNode( - graph, - shader, - conv2d_gwg, - conv2d_lwg, - // Inputs and Outputs - {{out, vkapi::kWrite}, {{in, arg_weight, arg_bias}, vkapi::kRead}}, - // Shader params buffers - param_buffers, - // Push Constants - {}, - // Specialization Constants - {utils::safe_downcast(groups_val)}, - // Resize Args - {weight_data, stride, padding, dilation, transposed, output_padding}, - // Resizing Logic - resize_conv2d_node)); -} - -void add_conv1d_node( - ComputeGraph& graph, - const ValueRef in, - const ValueRef weight, - const ValueRef bias, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef groups, - const ValueRef out_min, - const ValueRef out_max, - const ValueRef out, - const bool clamp_out) { - ValueRef arg_weight = prepack_standard( - graph, - weight, - graph.storage_type_of(out), - utils::kChannelsPacked, - /* passthrough = */ false); - ValueRef arg_bias = prepack_biases( - graph, - bias, - weight, - /*transposed = */ false, - /*storage_type = */ utils::kTexture3D, - /*memory_layout = */ utils::kWidthPacked); - - float out_min_val = 0.0f; - float out_max_val = 0.0f; - if (out_min != kDummyValueRef) { - out_min_val = graph.extract_scalar(out_min); - } - if (out_max != kDummyValueRef) { - out_max_val = graph.extract_scalar(out_max); - } - - const int64_t groups_val = graph.get_int(groups); - - const std::vector in_sizes = graph.sizes_of(in); - const std::vector weight_sizes = graph.sizes_of(arg_weight); - const std::vector out_sizes = graph.sizes_of(out); - - check_conv_args(graph, in, out); - - const int32_t in_channels = in_sizes.at(1); - const int32_t out_channels = weight_sizes.at(0); - const int32_t kernel_size = weight_sizes.at(2); - const int32_t stride_size = graph.get_int_list(stride)->at(0); - const int32_t padding_size = graph.get_int_list(padding)->at(0); - const int32_t dilation_size = graph.get_int_list(dilation)->at(0); - const int32_t in_group_size = static_cast(in_channels / groups_val); - const int32_t out_group_size = - static_cast(out_channels / groups_val); - - Kernel1dParams kernel_params = { - kernel_size, - stride_size, - padding_size, - dilation_size, - in_group_size, - out_group_size}; - - const OutputParams out_params = {out_min_val, out_max_val}; - - std::string kernel_name("conv1d"); - if (clamp_out) { - kernel_name += "_clamp"; - } - kernel_name.reserve(kShaderNameReserve); - - add_dtype_suffix(kernel_name, graph.dtype_of(out)); - - graph.execute_nodes().emplace_back(new DynamicDispatchNode( - graph, - VK_KERNEL_FROM_STR(kernel_name), - conv1d_gwg, - default_pick_lwg, - // Inputs and Outputs - {{out, vkapi::kWrite}, {{in, arg_weight, arg_bias}, vkapi::kRead}}, - // Shader params buffers - { - graph.logical_limits_ubo(out), - graph.sizes_ubo(in), - graph.create_params_buffer(kernel_params), - graph.create_params_buffer(out_params), - }, - // Push Constants - {}, - // Specialization Constants - {graph.hashed_layout_of(out), - graph.hashed_layout_of(in), - graph.hashed_layout_of(arg_weight), - graph.hashed_layout_of(arg_bias)}, - // Resize Args - {weight, stride, padding, dilation}, - // Resizing Logic - resize_conv1d_node)); -} - -void conv(ComputeGraph& graph, const std::vector& args) { - int64_t in_ndim = graph.dim_of(args[0]); - if (in_ndim == 4) { - if (args.size() == 10) { - // ordinary conv2d - return add_conv2d_node( - graph, - args[0], - args[1], - args[2], - args[3], - args[4], - args[5], - args[6], - args[7], - args[8], - /*out_min = */ kDummyValueRef, - /*out_max = */ kDummyValueRef, - args[9], - false); - } else { - // conv2d with clamp - return add_conv2d_node( - graph, - args[0], - args[1], - args[2], - args[3], - args[4], - args[5], - args[6], - args[7], - args[8], - args[9], - args[10], - args[11], - true); - } - } else { - // Conv1d path - if (graph.packed_dim_of(args[0]) == WHCN::kHeightDim) { - // Height-packed: route to optimized conv1d implementations - const auto weight_sizes = graph.sizes_of(args[1]); - const int64_t groups_val = graph.get_int(args[8]); - const bool is_pointwise = weight_sizes.at(2) == 1; - const bool is_depthwise = - groups_val == weight_sizes.at(0) && weight_sizes.at(1) == 1; - - // Build unified 10-arg vector: - // in, weight, bias, stride, padding, dilation, groups, - // output_min, output_max, out - // For non-clamp (args.size() == 10): output_min/max = kDummyValueRef - // For clamp (args.size() == 12): output_min/max from args[9]/args[10] - ValueRef output_min = kDummyValueRef; - ValueRef output_max = kDummyValueRef; - ValueRef out; - if (args.size() == 10) { - out = args[9]; - } else { - output_min = args[9]; - output_max = args[10]; - out = args[11]; - } - - std::vector conv1d_args = { - args[0], - args[1], - args[2], - args[3], - args[4], - args[5], - args[8], - output_min, - output_max, - out}; - - if (is_pointwise) { - VK_GET_OP_FN("et_vk.conv1d_pw.default")(graph, conv1d_args); - } else if (is_depthwise) { - VK_GET_OP_FN("et_vk.conv1d_dw.default")(graph, conv1d_args); - } else { - VK_THROW( - "Height-packed conv1d only supports pointwise (K=1) or " - "depthwise (groups=C)"); - } - return; - } - - // Existing channels-packed fallback - if (args.size() == 10) { - // ordinary conv1d - return add_conv1d_node( - graph, - args[0], - args[1], - args[2], - args[3], - args[4], - args[5], - args[8], - /*out_min = */ kDummyValueRef, - /*out_max = */ kDummyValueRef, - args[9], - false); - } else { - // conv1d with clamp - return add_conv1d_node( - graph, - args[0], - args[1], - args[2], - args[3], - args[4], - args[5], - args[8], - args[9], - args[10], - args[11], - true); - } - } -} - -REGISTER_OPERATORS { - VK_REGISTER_OP(aten.convolution.default, conv); - VK_REGISTER_OP(conv_with_clamp.default, conv); - VK_REGISTER_OP(et_vk.conv_with_clamp.default, conv); -} - -} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.h b/backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.h deleted file mode 100644 index e48d1e32a08..00000000000 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.h +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include -#include -#include - -namespace vkcompute { - -inline constexpr int64_t kQ8taConv2dIm2ColScratchBudgetBytes = 16 * 1024 * 1024; -inline constexpr int64_t kQ8taConv2dMaxRowsPerTile = 65535; - -struct Q8taConv2dStreamPlan final { - int64_t aligned_out_width; - int64_t rows_per_tile; - int64_t num_tiles; - int64_t scratch_bytes; - bool feasible; -}; - -Q8taConv2dStreamPlan make_q8ta_conv2d_stream_plan( - int64_t batch, - int64_t flattened_kernel_size, - int64_t out_height, - int64_t out_width, - int64_t scratch_budget_bytes); - -// max_buffer_bytes is Adapter::max_buffer_numel(), which returns -// maxStorageBufferRange in bytes (not elements) — directly comparable with -// the byte-denominated scratch budget. -Q8taConv2dStreamPlan make_q8ta_conv2d_stream_plan_for_device( - int64_t batch, - int64_t flattened_kernel_size, - int64_t out_height, - int64_t out_width, - uint64_t max_buffer_bytes); - -enum class ActivationType : uint32_t { - kNone = 0, - kRelu = 1, -}; - -ActivationType activation_type_from_string(const std::string& activation); - -bool q8ta_conv2d_check_packed_dim_info(const api::PackedDimInfo& info); - -bool q8ta_conv2d_check_4w4c_packed_dim_info(const api::PackedDimInfo& info); - -ValueRef prepack_quantized_conv2d_weight( - ComputeGraph& graph, - const QuantizationConfig& weight_quant_config, - const ValueRef weight_data, - const ValueRef input, - const ValueRef output, - const ValueRef groups, - const ValueRef kernel_size); - -ValueRef prepack_quantized_conv2d_weight( - ComputeGraph& graph, - const QuantizationConfig& weight_quant_config, - const ValueRef weight_data, - const ValueRef input, - const ValueRef output, - const ValueRef groups, - const ValueRef kernel_size); - -ValueRef prepack_quantized_conv2d_dw_weight( - ComputeGraph& graph, - const QuantizationConfig& weight_quant_config, - const ValueRef weight_data, - const ValueRef kernel_size); - -void add_q8ta_conv2d_dw_node( - ComputeGraph& graph, - const ValueRef packed_int8_input, - const ValueRef input_scale, - const ValueRef input_zp, - const ValueRef packed_weight, - const ValueRef packed_weight_sums, - const ValueRef packed_weight_scales, - const ValueRef output_scale, - const ValueRef output_zp, - const ValueRef bias_data, - const ValueRef packed_bias, - const ValueRef kernel_size, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef groups, - const uint32_t activation_type, - const ValueRef packed_int8_output); - -void add_conv2d_dw_q8ta_q8csw_q8to_4w4c_node( - ComputeGraph& graph, - const ValueRef packed_int8_input, - const ValueRef input_scale, - const ValueRef input_zp, - const ValueRef packed_weight, - const ValueRef packed_weight_sums, - const ValueRef packed_weight_scales, - const ValueRef output_scale, - const ValueRef output_zp, - const ValueRef bias_data, - const ValueRef packed_bias, - const ValueRef kernel_size, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef groups, - const ValueRef packed_int8_output); - -void add_q8ta_conv2d_node( - ComputeGraph& graph, - const ValueRef packed_int8_input, - const ValueRef packed_int8_input_im2col, - const ValueRef input_scale, - const ValueRef input_zp, - const ValueRef packed_weight, - const ValueRef packed_weight_sums, - const ValueRef packed_weight_scales, - const ValueRef output_scale, - const ValueRef output_zp, - const ValueRef bias_data, - const ValueRef packed_bias, - const ValueRef kernel_size, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef groups, - const uint32_t activation_type, - const ValueRef packed_int8_output); - -void add_q8ta_conv2d_pw_node( - ComputeGraph& graph, - const bool use_unsigned_dot, - const ValueRef packed_int8_input, - const ValueRef input_scale, - const ValueRef input_zp, - const ValueRef packed_weight, - const ValueRef packed_weight_sums, - const ValueRef packed_weight_scales, - const ValueRef output_scale, - const ValueRef output_zp, - const ValueRef bias_data, - const ValueRef packed_bias, - const uint32_t activation_type, - const ValueRef packed_int8_output, - const int32_t groups = 1, - const ValueRef conv_input = kDummyValueRef, - const ValueRef kernel_size = kDummyValueRef, - const ValueRef stride = kDummyValueRef, - const ValueRef padding = kDummyValueRef, - const ValueRef dilation = kDummyValueRef, - const bool is_im2col = false, - const ValueRef stream_row_offset_ref = kDummyValueRef, - const ValueRef max_im2col_rows_ref = kDummyValueRef); - -constexpr int64_t kMaxUnsignedDotAccumulatorBytes = 33025; - -bool can_use_unsigned_pw_dot( - const vkapi::Adapter& adapter, - int64_t k_per_group); - -void q8ta_conv2d_pw_impl( - ComputeGraph& graph, - bool use_unsigned_dot, - const std::vector& args); - -void q8ta_conv2d_pw(ComputeGraph& graph, const std::vector& args); - -std::vector calculate_q8ta_im2col_sizes( - ComputeGraph* graph, - const ValueRef& input, - const ValueRef& output, - const ValueRef& kernel_size, - const ValueRef& groups); - -void add_q8ta_im2col_node( - ComputeGraph& graph, - const ValueRef packed_int8_input, - const ValueRef kernel_size, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef groups, - const ValueRef packed_int8_output, - const ValueRef packed_int8_im2col, - const int32_t zp, - const ValueRef stream_row_offset_ref, - const ValueRef max_im2col_rows_ref = kDummyValueRef); - -void q8ta_conv2d_im2col(ComputeGraph& graph, const std::vector& args); - -void q8ta_conv2d_im2col_impl( - ComputeGraph& graph, - bool use_unsigned_dot, - const std::vector& args); - -void q8ta_conv2d_general( - ComputeGraph& graph, - const std::vector& args); - -// Transposed convolution - -void q8ta_conv2d_transposed( - ComputeGraph& graph, - const std::vector& args); - -} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taLinear.cpp b/backends/vulkan/runtime/graph/ops/impl/Q8taLinear.cpp index 683b7b5ce07..ea1a87b910d 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taLinear.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/Q8taLinear.cpp @@ -11,9 +11,9 @@ #include #include -#include #include #include +#include #include namespace vkcompute { diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.cpp b/backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.cpp index 7a63d33d546..a3944641839 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.cpp @@ -11,9 +11,9 @@ #include #include -#include #include #include +#include #include namespace vkcompute { diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.cpp new file mode 100644 index 00000000000..4bbb2c56911 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.cpp @@ -0,0 +1,84 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include + +#include +#include + +#include + +namespace vkcompute { + +ValueRef prepack_biases( + ComputeGraph& graph, + const ValueRef vref, + const ValueRef weight, + const bool transposed, + const utils::StorageType storage_type, + const utils::GPUMemoryLayout memory_layout) { + auto sizes = graph.sizes_of(weight); + const int64_t out_channels = transposed ? sizes.at(1) : sizes.at(0); + + ValueRef v = graph.add_tensor( + {out_channels}, graph.dtype_of(weight), storage_type, memory_layout); + + vkapi::ShaderInfo shader = + get_nchw_to_tensor_shader(graph, v, graph.get_staging_dtype_for(weight)); + + vkapi::ParamsBindList param_buffers = {}; + if (graph.is_buffer_storage(v)) { + param_buffers.append(graph.buffer_meta_ubo(v)); + } else { + param_buffers.append(graph.texture_meta_ubo(v)); + } + + std::vector pcs; + if (graph.is_buffer_storage(v)) { + pcs = {graph.sizes_pc_of(v), graph.strides_pc_of(v), graph.numel_pc_of(v)}; + } + + graph.prepack_nodes().emplace_back(new PrepackNode( + graph, + shader, + graph.create_gwg(v), + graph.create_lwg(v), + vref, + v, + param_buffers, + // Specialization constants + {graph.hashed_layout_of(v)}, + pcs)); + + return v; +} + +void check_conv_args( + ComputeGraph& graph, + const ValueRef in, + const ValueRef out) { + VK_CHECK_COND(graph.packed_dim_of(in) == WHCN::kChannelsDim); + VK_CHECK_COND(graph.packed_dim_of(out) == WHCN::kChannelsDim); +} + +void conv(ComputeGraph& graph, const std::vector& args) { + if (graph.dim_of(args[0]) == 4) { + return conv2d_impl(graph, args); + } + return conv1d_impl(graph, args); +} + +REGISTER_OPERATORS { + VK_REGISTER_OP(aten.convolution.default, conv); + VK_REGISTER_OP(conv_with_clamp.default, conv); + VK_REGISTER_OP(et_vk.conv_with_clamp.default, conv); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.h b/backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.h new file mode 100644 index 00000000000..08bf5bd5c9c --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +ValueRef prepack_biases( + ComputeGraph& graph, + const ValueRef vref, + const ValueRef weight, + const bool transposed, + const utils::StorageType storage_type, + const utils::GPUMemoryLayout memory_layout); + +void check_conv_args( + ComputeGraph& graph, + const ValueRef in, + const ValueRef out); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.cpp new file mode 100644 index 00000000000..1f71e009d32 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.cpp @@ -0,0 +1,100 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include + +#include + +namespace vkcompute { + +void conv1d_impl(ComputeGraph& graph, const std::vector& args) { + if (graph.packed_dim_of(args[0]) == WHCN::kHeightDim) { + // Height-packed: route to optimized conv1d implementations + const auto weight_sizes = graph.sizes_of(args[1]); + const int64_t groups_val = graph.get_int(args[8]); + const bool is_pointwise = weight_sizes.at(2) == 1; + const bool is_depthwise = + groups_val == weight_sizes.at(0) && weight_sizes.at(1) == 1; + + // Build unified 10-arg vector: + // in, weight, bias, stride, padding, dilation, groups, + // output_min, output_max, out + // For non-clamp (args.size() == 10): output_min/max = kDummyValueRef + // For clamp (args.size() == 12): output_min/max from args[9]/args[10] + ValueRef output_min = kDummyValueRef; + ValueRef output_max = kDummyValueRef; + ValueRef out; + if (args.size() == 10) { + out = args[9]; + } else { + output_min = args[9]; + output_max = args[10]; + out = args[11]; + } + + std::vector conv1d_args = { + args[0], + args[1], + args[2], + args[3], + args[4], + args[5], + args[8], + output_min, + output_max, + out}; + + if (is_pointwise) { + VK_GET_OP_FN("et_vk.conv1d_pw.default")(graph, conv1d_args); + } else if (is_depthwise) { + VK_GET_OP_FN("et_vk.conv1d_dw.default")(graph, conv1d_args); + } else { + VK_THROW( + "Height-packed conv1d only supports pointwise (K=1) or " + "depthwise (groups=C)"); + } + return; + } + + // Existing channels-packed fallback + if (args.size() == 10) { + // ordinary conv1d + return add_conv1d_direct_node( + graph, + args[0], + args[1], + args[2], + args[3], + args[4], + args[5], + args[8], + /*out_min = */ kDummyValueRef, + /*out_max = */ kDummyValueRef, + args[9], + false); + } else { + // conv1d with clamp + return add_conv1d_direct_node( + graph, + args[0], + args[1], + args[2], + args[3], + args[4], + args[5], + args[8], + args[9], + args[10], + args[11], + true); + } +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.h new file mode 100644 index 00000000000..f6030128587 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +void conv1d_impl(ComputeGraph& graph, const std::vector& args); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv1dDW.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDW.cpp similarity index 97% rename from backends/vulkan/runtime/graph/ops/impl/Conv1dDW.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDW.cpp index ba59090c124..58971984821 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Conv1dDW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDW.cpp @@ -150,7 +150,7 @@ void add_conv1d_dw_node( // Args: in, weight, bias, stride, padding, dilation, groups, // output_min, output_max, out // output_min and output_max may be kDummyValueRef (no clamp). -void conv1d_dw(ComputeGraph& graph, const std::vector& args) { +void conv1d_dw_impl(ComputeGraph& graph, const std::vector& args) { ValueRef in = args[0]; ValueRef weight = args[1]; ValueRef bias = args[2]; @@ -182,7 +182,7 @@ void conv1d_dw(ComputeGraph& graph, const std::vector& args) { } REGISTER_OPERATORS { - VK_REGISTER_OP(et_vk.conv1d_dw.default, conv1d_dw); + VK_REGISTER_OP(et_vk.conv1d_dw.default, conv1d_dw_impl); } } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.cpp new file mode 100644 index 00000000000..bdd6f8da6e9 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.cpp @@ -0,0 +1,179 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include +#include + +#include + +#include + +namespace vkcompute { + +void resize_conv1d_node( + ComputeGraph* graph, + const std::vector& args, + const std::vector& extra_args) { + const ValueRef out = args.at(0).refs.at(0); + const ValueRef self = args.at(1).refs.at(0); + TensorRefPtr weight_ref = graph->get_tref(extra_args.at(0)); + + const int64_t stride_size = graph->get_int_list(extra_args.at(1))->at(0); + const int64_t padding_size = graph->get_int_list(extra_args.at(2))->at(0); + const int64_t dilation_size = graph->get_int_list(extra_args.at(3))->at(0); + + const std::vector& weight_sizes = weight_ref->sizes; + + const std::vector in_sizes = graph->sizes_of(self); + const size_t ndim = in_sizes.size(); + std::vector new_out_sizes(ndim); + + const int64_t kernel_size = weight_sizes.at(2); + const int64_t in_length = in_sizes.at(2); + + new_out_sizes.at(0) = in_sizes.at(0); + new_out_sizes.at(1) = weight_sizes.at(0); + new_out_sizes.at(2) = calc_out_size( + in_length, kernel_size, stride_size, padding_size, dilation_size, false); + + graph->virtual_resize(out, new_out_sizes); +} + +struct OutputParams final { + float out_min; + float out_max; +}; + +// Custom global workgroup size function for conv1d +GlobalWorkGrid conv1d_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + (void)shader; + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + + return GlobalWorkGrid( + {// out length + graph->size_at(-1, out), + // out channels + static_cast(graph->size_at(-2, out)), + // out batches + utils::div_up_4(graph->size_at(-3, out))}, + kTiledWorkGrid); +} + +void add_conv1d_direct_node( + ComputeGraph& graph, + const ValueRef in, + const ValueRef weight, + const ValueRef bias, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef groups, + const ValueRef out_min, + const ValueRef out_max, + const ValueRef out, + const bool clamp_out) { + ValueRef arg_weight = prepack_standard( + graph, + weight, + graph.storage_type_of(out), + utils::kChannelsPacked, + /* passthrough = */ false); + ValueRef arg_bias = prepack_biases( + graph, + bias, + weight, + /*transposed = */ false, + /*storage_type = */ utils::kTexture3D, + /*memory_layout = */ utils::kWidthPacked); + + float out_min_val = 0.0f; + float out_max_val = 0.0f; + if (out_min != kDummyValueRef) { + out_min_val = graph.extract_scalar(out_min); + } + if (out_max != kDummyValueRef) { + out_max_val = graph.extract_scalar(out_max); + } + + const int64_t groups_val = graph.get_int(groups); + + const std::vector in_sizes = graph.sizes_of(in); + const std::vector weight_sizes = graph.sizes_of(arg_weight); + const std::vector out_sizes = graph.sizes_of(out); + + check_conv_args(graph, in, out); + + const int32_t in_channels = utils::safe_downcast(in_sizes.at(1)); + const int32_t out_channels = + utils::safe_downcast(weight_sizes.at(0)); + const int32_t kernel_size = utils::safe_downcast(weight_sizes.at(2)); + const int32_t stride_size = + utils::safe_downcast(graph.get_int_list(stride)->at(0)); + const int32_t padding_size = + utils::safe_downcast(graph.get_int_list(padding)->at(0)); + const int32_t dilation_size = + utils::safe_downcast(graph.get_int_list(dilation)->at(0)); + const int32_t in_group_size = + utils::safe_downcast(in_channels / groups_val); + const int32_t out_group_size = + utils::safe_downcast(out_channels / groups_val); + + Kernel1dParams kernel_params = { + kernel_size, + stride_size, + padding_size, + dilation_size, + in_group_size, + out_group_size}; + + const OutputParams out_params = {out_min_val, out_max_val}; + + std::string kernel_name("conv1d"); + if (clamp_out) { + kernel_name += "_clamp"; + } + kernel_name.reserve(kShaderNameReserve); + + add_dtype_suffix(kernel_name, graph.dtype_of(out)); + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + VK_KERNEL_FROM_STR(kernel_name), + conv1d_gwg, + default_pick_lwg, + // Inputs and Outputs + {{out, vkapi::kWrite}, {{in, arg_weight, arg_bias}, vkapi::kRead}}, + // Shader params buffers + { + graph.logical_limits_ubo(out), + graph.sizes_ubo(in), + graph.create_params_buffer(kernel_params), + graph.create_params_buffer(out_params), + }, + // Push Constants + {}, + // Specialization Constants + {graph.hashed_layout_of(out), + graph.hashed_layout_of(in), + graph.hashed_layout_of(arg_weight), + graph.hashed_layout_of(arg_bias)}, + // Resize Args + {weight, stride, padding, dilation}, + // Resizing Logic + resize_conv1d_node)); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.h new file mode 100644 index 00000000000..9800330b9f8 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +void add_conv1d_direct_node( + ComputeGraph& graph, + const ValueRef in, + const ValueRef weight, + const ValueRef bias, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef groups, + const ValueRef out_min, + const ValueRef out_max, + const ValueRef out, + const bool clamp_out); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv1dPW.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dPW.cpp similarity index 98% rename from backends/vulkan/runtime/graph/ops/impl/Conv1dPW.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dPW.cpp index de2fcdbb806..225a5118688 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Conv1dPW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dPW.cpp @@ -211,7 +211,7 @@ void add_conv1d_pw_node( // Args: in, weight, bias, stride, padding, dilation, groups, // output_min, output_max, out // output_min and output_max may be kDummyValueRef (no clamp). -void conv1d_pw(ComputeGraph& graph, const std::vector& args) { +void conv1d_pw_impl(ComputeGraph& graph, const std::vector& args) { ValueRef in = args[0]; ValueRef weight = args[1]; ValueRef bias = args[2]; @@ -236,7 +236,7 @@ void conv1d_pw(ComputeGraph& graph, const std::vector& args) { } REGISTER_OPERATORS { - VK_REGISTER_OP(et_vk.conv1d_pw.default, conv1d_pw); + VK_REGISTER_OP(et_vk.conv1d_pw.default, conv1d_pw_impl); } } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2d.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2d.cpp new file mode 100644 index 00000000000..ff4f8173780 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2d.cpp @@ -0,0 +1,234 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include +#include + +#include + +namespace vkcompute { + +void resize_conv2d_node( + ComputeGraph* graph, + const std::vector& args, + const std::vector& extra_args) { + const ValueRef out = args.at(0).refs.at(0); + const ValueRef self = args.at(1).refs.at(0); + + size_t ndim = graph->dim_of(self); + std::vector new_out_sizes(ndim); + const bool transposed = graph->get_bool(extra_args.at(4)); + + std::vector self_sizes = graph->sizes_of(self); + // Batch, Channel + if (ndim == 4) { + new_out_sizes.at(ndim - 4) = self_sizes.at(ndim - 4); + } + + TensorRefPtr weight_ref = graph->get_tref(extra_args.at(0)); + const auto& weight_sizes = weight_ref->sizes; + new_out_sizes.at(ndim - 3) = + transposed ? weight_sizes.at(ndim - 3) : weight_sizes.at(ndim - 4); + + // Height, Width + const auto& new_out_sizes_hw = calc_out_sizes_hw( + *graph, + self_sizes, + extra_args.at(0), + /*kernel_size_only = */ false, + {extra_args.at(1), extra_args.at(2), extra_args.at(3), extra_args.at(5)}, + transposed); + new_out_sizes.at(ndim - 2) = new_out_sizes_hw.at(0); + new_out_sizes.at(ndim - 1) = new_out_sizes_hw.at(1); + + graph->virtual_resize(out, new_out_sizes); +} + +Conv2dMethod get_conv2d_method( + ComputeGraph& graph, + const ValueRef weight, + const int64_t groups, + const bool transposed) { + const auto weight_sizes = graph.sizes_of(weight); + if (!transposed && weight_sizes.at(0) == groups && weight_sizes.at(1) == 1) { + return Conv2dMethod::Depthwise; + } + if (transposed) { + return Conv2dMethod::Transposed; + } + if (weight_sizes.at(2) == 1 && weight_sizes.at(3) == 1) { + return Conv2dMethod::Pointwise; + } + return Conv2dMethod::SlidingWindow; +} + +// Decide whether a SlidingWindow conv2d should be computed via the +// im2col + GEMM path (conv2d_gemm_impl) instead of the direct convolution +// shader. Across 26 configs on Mali-G715 (buffer path) and Adreno SM8650 +// (texture path): FP32 cases were numerically verified against the reference; +// FP16 cases were routing/dispatch-validated only (the reference is float-only +// for the large shapes, so FP16 outputs were not numerically checked). +// +// Only called for SlidingWindow conv2d (1x1 is routed to conv2d_pw and +// Depthwise/Transposed are handled before the call site). +// +// Preconditions (fall back to direct conv if any fail — the im2col path is +// either not applicable or not beneficial): +// - groups == 1 +// - dilation == 1 (all dims) +// +// Selection rule: use im2col once the output channel count is large enough to +// amortize the fixed ~N*K_total im2col gather cost. +constexpr int64_t kIm2colMinCOut = 128; + +// A cheap gather is a second, independent reason to take im2col. The gather +// materializes an N x K_total matrix before the GEMM runs, and a large gather +// is fine when c_out is large because the GEMM reads it back c_out times. When +// c_out is small it is only worth paying if the matrix is small outright, which +// is where the direct shader on Mali loses badly: an 80x80 3x3 conv wants ~4M +// elements, a 640x640 9x9 conv wants ~100M. +constexpr int64_t kIm2colMaxCheapGatherElements = 32 * 1024 * 1024; + +bool should_use_conv2d_im2col( + ComputeGraph& graph, + const ValueRef weight_data, + const int64_t groups_val, + const Kernel2dParams& kernel_params, + const ValueRef out) { + if (groups_val != 1) { + return false; + } + if (kernel_params.dilation[0] != 1 || kernel_params.dilation[1] != 1) { + return false; + } + const auto weight_sizes = graph.sizes_of(weight_data); + const int64_t c_out = weight_sizes.at(0); + const int64_t k_total = + weight_sizes.at(1) * weight_sizes.at(2) * weight_sizes.at(3); + + const auto out_sizes = graph.sizes_of(out); + const size_t ndim = out_sizes.size(); + const int64_t n = out_sizes.at(ndim - 1) * out_sizes.at(ndim - 2); + return c_out >= kIm2colMinCOut || + (graph.device_is_mali() && n * k_total <= kIm2colMaxCheapGatherElements); +} + +void conv2d_impl(ComputeGraph& graph, const std::vector& args) { + const ValueRef in = args[0]; + const ValueRef weight_data = args[1]; + const ValueRef bias = args[2]; + const ValueRef stride = args[3]; + const ValueRef padding = args[4]; + const ValueRef dilation = args[5]; + const ValueRef transposed = args[6]; + const ValueRef output_padding = args[7]; + const ValueRef groups = args[8]; + // conv2d with clamp passes out_min and out_max before the output + const bool clamp_out = args.size() != 10; + const ValueRef out_min = clamp_out ? args[9] : kDummyValueRef; + const ValueRef out_max = clamp_out ? args[10] : kDummyValueRef; + const ValueRef out = clamp_out ? args[11] : args[9]; + + const bool transposed_val = graph.get_bool(transposed); + + float out_min_val = 0.0f; + float out_max_val = 0.0f; + if (out_min != kDummyValueRef) { + out_min_val = graph.extract_scalar(out_min); + } + if (out_max != kDummyValueRef) { + out_max_val = graph.extract_scalar(out_max); + } + + const int64_t groups_val = graph.get_int(groups); + + const Conv2dMethod method = + get_conv2d_method(graph, weight_data, groups_val, transposed_val); + + // Use tiled path for all pointwise conv2d + if (method == Conv2dMethod::Pointwise) { + return conv2d_pw_impl( + graph, + in, + weight_data, + bias, + stride, + padding, + out, + transposed_val, + clamp_out, + out_min_val, + out_max_val); + } + + if (method == Conv2dMethod::Depthwise) { + return conv2d_dw_impl( + graph, + in, + weight_data, + bias, + stride, + padding, + dilation, + out, + clamp_out, + out_min_val, + out_max_val); + } + + const Kernel2dParams kernel_params = create_kernel2d_params( + graph, + weight_data, + /*kernel_size_only = */ false, + stride, + padding, + dilation); + + // SlidingWindow conv2d: route to the im2col + GEMM path when the heuristic + // indicates it is beneficial, falling back to the direct convolution shader + // otherwise. + const bool use_im2col = + method == Conv2dMethod::SlidingWindow && + should_use_conv2d_im2col( + graph, weight_data, groups_val, kernel_params, out); + if (use_im2col) { + return conv2d_gemm_impl( + graph, + in, + weight_data, + bias, + stride, + padding, + dilation, + out, + clamp_out, + out_min_val, + out_max_val); + } + + return conv2d_direct_impl( + graph, + in, + weight_data, + bias, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + out, + clamp_out, + out_min_val, + out_max_val); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Convolution.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2d.h similarity index 50% rename from backends/vulkan/runtime/graph/ops/impl/Convolution.h rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2d.h index 12e4733e67a..446859c363d 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Convolution.h +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2d.h @@ -12,18 +12,12 @@ namespace vkcompute { -ValueRef prepack_biases( - ComputeGraph& graph, - const ValueRef vref, - const ValueRef weight, - const bool transposed, - const utils::StorageType storage_type, - const utils::GPUMemoryLayout memory_layout); - -void check_conv_args( - ComputeGraph& graph, - const ValueRef in, - const ValueRef out); +enum class Conv2dMethod : uint8_t { + Depthwise, + Pointwise, + SlidingWindow, + Transposed, +}; void conv2d_pw_impl( ComputeGraph& graph, @@ -56,26 +50,6 @@ void resize_conv2d_node( const std::vector& args, const std::vector& extra_args); -// `force_direct` overrides the im2col-vs-direct routing heuristic: when true, -// a SlidingWindow conv2d always takes the direct sliding-window path, -// bypassing should_use_conv2d_im2col(). The default (false) preserves the -// production routing exactly. Pointwise / Depthwise / Transposed methods are -// unaffected by this flag. -void add_conv2d_node( - ComputeGraph& graph, - const ValueRef in, - const ValueRef weight_data, - const ValueRef bias, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef transposed, - const ValueRef output_padding, - const ValueRef groups, - const ValueRef out_min, - const ValueRef out_max, - const ValueRef out, - const bool clamp_out, - const bool force_direct = false); +void conv2d_impl(ComputeGraph& graph, const std::vector& args); } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv2dDW.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDW.cpp similarity index 98% rename from backends/vulkan/runtime/graph/ops/impl/Conv2dDW.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDW.cpp index a21f6c93e4e..0ac6a56d7ad 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Conv2dDW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDW.cpp @@ -6,10 +6,11 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include +#include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.cpp new file mode 100644 index 00000000000..a476d03a900 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.cpp @@ -0,0 +1,402 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include +#include + +#include + +#include + +namespace vkcompute { + +vkapi::ShaderInfo get_conv2d_shader( + ComputeGraph& graph, + const ValueRef out, + const bool prepack_weights, + const Conv2dMethod method, + const ValueRef weight, + const bool clamp_out = false, + const bool stride_equals_dilation = false, + const bool stride_1_padding_0 = false) { + (void)stride_equals_dilation; + std::string kernel_name; + kernel_name.reserve(kShaderNameReserve); + switch (method) { + case Conv2dMethod::Depthwise: + kernel_name = "conv2d_dw"; + break; + case Conv2dMethod::Pointwise: + if (prepack_weights) { + kernel_name = "conv2d"; + } else { + kernel_name = stride_1_padding_0 ? "conv2d_pw_s1p0" : "conv2d_pw"; + } + break; + case Conv2dMethod::SlidingWindow: + kernel_name = "conv2d"; + break; + case Conv2dMethod::Transposed: + kernel_name = "conv_transpose2d"; + break; + } + if (prepack_weights) { + kernel_name += "_prepack_weights"; + } else if (clamp_out) { + kernel_name += "_clamp"; + } + add_dtype_suffix(kernel_name, graph.dtype_of(out)); + + if (prepack_weights) { + add_dtype_suffix(kernel_name, graph.get_staging_dtype_for(weight)); + } + + return VK_KERNEL_FROM_STR(kernel_name); +} + +std::vector get_final_sizes( + const std::vector& original_sizes, + const Conv2dMethod method) { + int64_t batch_padded = utils::align_up_4(utils::val_at(-4, original_sizes)); + int64_t channels_padded = + utils::align_up_4(utils::val_at(-3, original_sizes)); + int64_t height = utils::val_at(-2, original_sizes); + int64_t width = utils::val_at(-1, original_sizes); + + switch (method) { + case Conv2dMethod::Depthwise: + return std::vector{4, batch_padded / 4, height * width}; + case Conv2dMethod::Pointwise: + case Conv2dMethod::SlidingWindow: + return std::vector{ + 4, batch_padded * height / 4, channels_padded * width}; + case Conv2dMethod::Transposed: + return std::vector{ + 4, channels_padded * height / 4, batch_padded * width}; + } +} + +ValueRef prepack_weights( + ComputeGraph& graph, + const ValueRef vref, + const Conv2dMethod method) { + const auto original_sizes = graph.sizes_of(vref); + const auto final_sizes = get_final_sizes(original_sizes, method); + + ValueRef v = graph.add_tensor( + final_sizes, + graph.dtype_of(vref), + utils::kTexture2D, + utils::kChannelsPacked); + + vkapi::ShaderInfo shader = + get_conv2d_shader(graph, v, /*prepack_weights = */ true, method, vref); + + const auto original_sizes_pc = + utils::make_ivec4(original_sizes, /*reverse = */ true); + graph.prepack_nodes().emplace_back(new PrepackNode( + graph, + shader, + graph.create_gwg(v), + graph.create_lwg(v), + vref, + v, + {}, + // Specialization constants + {graph.packed_dim_of(v)}, + {graph.sizes_pc_of(v), + PushConstantDataInfo(&original_sizes_pc, sizeof(original_sizes_pc))})); + + return v; +} + +namespace { + +struct Conv2dParams final { + utils::ivec2 overlay_region; + int in_group_size; +}; + +} // namespace + +struct OutputParams final { + float out_min; + float out_max; +}; + +Conv2dParams create_conv2d_params( + ComputeGraph& graph, + const ValueRef weight, + const Kernel2dParams& p, + const bool transposed) { + const auto& overlay_region = utils::make_ivec2({ + p.kernel_size[0] + (p.kernel_size[0] - 1) * (p.dilation[0] - 1), + p.kernel_size[1] + (p.kernel_size[1] - 1) * (p.dilation[1] - 1), + }); + const auto weight_sizes = graph.sizes_of(weight); + const int32_t in_group_size = utils::safe_downcast( + utils::align_up_4(transposed ? weight_sizes.at(0) : weight_sizes.at(1))); + return {overlay_region, in_group_size}; +} + +void check_conv2d_params(const Kernel2dParams& p, const bool transposed) { + if (transposed) { + if (p.dilation[0] > 1 || p.dilation[1] > 1) { + VK_THROW( + "aten.convolution.default: transposed = true, dilation > 1 is not supported yet!"); + } + } +} + +GlobalWorkGrid create_conv2d_gwg( + ComputeGraph& graph, + const Conv2dMethod method, + const ValueRef out, + const ValueRef weight_data, + const bool stride_equals_dilation) { + (void)weight_data; + (void)stride_equals_dilation; + if (method == Conv2dMethod::Pointwise) { + const utils::uvec3 image_extents = graph.logical_limits_of(out); + return GlobalWorkGrid( + {utils::div_up(image_extents[0u], 1u), + utils::div_up(image_extents[1u], 4u), + image_extents[2u]}, + kTiledWorkGrid); + } else { + return graph.create_gwg(out); + } +} + +// Determines which convolution method a dispatch uses. +// +// Depthwise and transposed convolutions have shader names of their own, but +// the name alone cannot separate pointwise from sliding window: the sliding +// window shader is itself named "conv2d", and a pointwise convolution also +// takes that name when its weights are prepacked. Those two are therefore +// separated by the weight's spatial extent. Shared by the global and local +// workgroup size functions below so that the two cannot disagree about the +// same dispatch. +Conv2dMethod infer_conv2d_method_from_shader( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const ValueRef weight_data) { + const std::string& kernel_name = shader.kernel_name; + // Checked before the plain "conv2d" test below, which "conv2d_dw" and + // "conv2d_pw" would otherwise match too. + if (kernel_name.find("conv2d_dw") != std::string::npos) { + return Conv2dMethod::Depthwise; + } + if (kernel_name.find("conv2d_pw") != std::string::npos) { + return Conv2dMethod::Pointwise; + } + if (kernel_name.find("conv_transpose2d") != std::string::npos) { + return Conv2dMethod::Transposed; + } + const auto& weight_sizes = graph->get_tref(weight_data)->sizes; + if (weight_sizes.at(2) == 1 && weight_sizes.at(3) == 1) { + return Conv2dMethod::Pointwise; + } + return Conv2dMethod::SlidingWindow; +} + +// Custom global workgroup size function for conv2d +GlobalWorkGrid conv2d_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + const ValueRef out = args.at(0).refs.at(0); + const ValueRef weight_data = resize_args.at(0); + + const Conv2dMethod method = + infer_conv2d_method_from_shader(graph, shader, weight_data); + + // Determine stride_equals_dilation from shader name + bool stride_equals_dilation = + shader.kernel_name.find("_sned") == std::string::npos; + + const GlobalWorkGrid wg_size = create_conv2d_gwg( + *graph, method, out, weight_data, stride_equals_dilation); + + if (method == Conv2dMethod::Pointwise) { + utils::uvec3 pointwise_wg_size = {wg_size[0] * wg_size[1], wg_size[2], 1u}; + + if (shader.kernel_name.find("s1p0") != std::string::npos) { + pointwise_wg_size[0] *= 4; + } + return GlobalWorkGrid(pointwise_wg_size, kTiledWorkGrid); + } + + return wg_size; +} + +// Custom local workgroup size function for conv2d +LocalWorkGroup conv2d_lwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args, + const std::vector& resize_args) { + (void)args; + + const ValueRef weight_data = resize_args.at(0); + const Conv2dMethod method = + infer_conv2d_method_from_shader(graph, shader, weight_data); + + if (method == Conv2dMethod::Pointwise) { + uint32_t lwg_y = 1; + if (gwg[1] % 8 == 0) { + lwg_y = 8; + } else if (gwg[1] % 4 == 0) { + lwg_y = 4; + } else if (gwg[1] % 2 == 0) { + lwg_y = 2; + } + return LocalWorkGroup(64u / lwg_y, lwg_y, 1u); + } else { + return graph->create_lwg(gwg); + } +} + +void add_conv2d_direct_node( + ComputeGraph& graph, + const ValueRef in, + const ValueRef arg_weight, + const ValueRef arg_bias, + const ValueRef weight_data, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef transposed, + const ValueRef output_padding, + const ValueRef out, + const vkapi::ShaderInfo& shader, + const Kernel2dParams& kernel_params, + const Conv2dParams& extra_params, + const OutputParams& out_params, + const int64_t groups_val) { + vkapi::ParamsBindList param_buffers = { + graph.logical_limits_ubo(out), + graph.sizes_ubo(in), + graph.create_params_buffer(kernel_params), + graph.create_params_buffer(extra_params), + graph.create_params_buffer(out_params), + }; + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + shader, + conv2d_gwg, + conv2d_lwg, + // Inputs and Outputs + {{out, vkapi::kWrite}, {{in, arg_weight, arg_bias}, vkapi::kRead}}, + // Shader params buffers + param_buffers, + // Push Constants + {}, + // Specialization Constants + {utils::safe_downcast(groups_val)}, + // Resize Args + {weight_data, stride, padding, dilation, transposed, output_padding}, + // Resizing Logic + resize_conv2d_node)); +} + +void conv2d_direct_impl( + ComputeGraph& graph, + const ValueRef in, + const ValueRef weight_data, + const ValueRef bias, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef transposed, + const ValueRef output_padding, + const ValueRef groups, + const ValueRef out, + const bool clamp_out, + const float out_min_val, + const float out_max_val) { + const bool transposed_val = graph.get_bool(transposed); + const int64_t groups_val = graph.get_int(groups); + const Conv2dMethod method = + transposed_val ? Conv2dMethod::Transposed : Conv2dMethod::SlidingWindow; + + const Kernel2dParams kernel_params = create_kernel2d_params( + graph, + weight_data, + /*kernel_size_only = */ false, + stride, + padding, + dilation); + + ValueRef arg_weight = prepack_weights(graph, weight_data, method); + ValueRef arg_bias = prepack_biases( + graph, + bias, + weight_data, + transposed_val, + /* storage_type = */ utils::kTexture2D, + /* memory_layout = */ utils::kWidthPacked); + + const std::vector in_sizes = graph.sizes_of(in); + if (in_sizes.at(0) > 1) { + VK_THROW("conv2d: input batch size > 1 is not supported yet!"); + } + + check_conv_args(graph, in, out); + + Conv2dParams extra_params = + create_conv2d_params(graph, weight_data, kernel_params, transposed_val); + + const bool stride_equals_dilation = + (kernel_params.stride[0] == kernel_params.dilation[0] && + kernel_params.stride[1] == kernel_params.dilation[1]); + + const bool stride_1_padding_0 = + (kernel_params.stride[0] == 1 && kernel_params.stride[1] == 1 && + kernel_params.padding[0] == 0 && kernel_params.padding[1] == 0); + + OutputParams out_params = {out_min_val, out_max_val}; + + check_conv2d_params(kernel_params, transposed_val); + + vkapi::ShaderInfo shader = get_conv2d_shader( + graph, + out, + /*prepack_weights = */ false, + method, + weight_data, + clamp_out, + stride_equals_dilation, + stride_1_padding_0); + + add_conv2d_direct_node( + graph, + in, + arg_weight, + arg_bias, + weight_data, + stride, + padding, + dilation, + transposed, + output_padding, + out, + shader, + kernel_params, + extra_params, + out_params, + groups_val); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.h new file mode 100644 index 00000000000..ef3ea388044 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +// Dispatches the direct conv2d shader, or the transposed conv2d shader when +// `transposed` is true; the two share weight prepacking and dispatch logic. +void conv2d_direct_impl( + ComputeGraph& graph, + const ValueRef in, + const ValueRef weight_data, + const ValueRef bias, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef transposed, + const ValueRef output_padding, + const ValueRef groups, + const ValueRef out, + const bool clamp_out, + const float out_min_val, + const float out_max_val); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv2dPW.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dPW.cpp similarity index 97% rename from backends/vulkan/runtime/graph/ops/impl/Conv2dPW.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dPW.cpp index 3c227448cf2..736bb306f77 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Conv2dPW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dPW.cpp @@ -6,10 +6,11 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include +#include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/ConvolutionUtils.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/ConvolutionUtils.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/ConvolutionUtils.h rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/ConvolutionUtils.h diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv2dGemm.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dGemm.cpp similarity index 98% rename from backends/vulkan/runtime/graph/ops/impl/Conv2dGemm.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dGemm.cpp index 6285dc92d4a..55c6582335b 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Conv2dGemm.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dGemm.cpp @@ -6,13 +6,13 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include -#include -#include #include +#include +#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv2dGemm.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dGemm.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Conv2dGemm.h rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dGemm.h diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv2dIm2Col.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dIm2Col.cpp similarity index 98% rename from backends/vulkan/runtime/graph/ops/impl/Conv2dIm2Col.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dIm2Col.cpp index e1bbded799e..e4dcf6592c2 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Conv2dIm2Col.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dIm2Col.cpp @@ -6,7 +6,7 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv2dIm2Col.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dIm2Col.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Conv2dIm2Col.h rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dIm2Col.h diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dRoute.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2d.cpp similarity index 70% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2dRoute.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2d.cpp index 780dfcbfbf3..290207295ca 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dRoute.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2d.cpp @@ -6,9 +6,13 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include -#include +#include + +#include +#include +#include #include #include @@ -144,4 +148,55 @@ bool should_use_q8ta_conv2d_im2col(const Q8taConv2dRouteParams& params) { (params.in_channels_per_group >= 32 || spatial_out <= 4096); } +void q8ta_conv2d(ComputeGraph& graph, const std::vector& args) { + const ValueRef input = args.at(0); + const ValueRef kernel_size_ref = args.at(9); + const ValueRef groups_ref = args.at(13); + const ValueRef output = args.at(15); + + const int64_t groups = graph.extract_scalar(groups_ref); + // Valid models always carry groups >= 1; fail fast on corrupt input + // instead of dividing channel counts by zero downstream (both this + // dispatcher and q8ta_conv2d_general_impl divide by groups). + VK_CHECK_COND(groups > 0, "q8ta_conv2d requires groups >= 1"); + const int64_t in_channels = graph.size_at(-3, input); + const int64_t in_channels_per_group = in_channels / groups; + const int64_t batch = graph.size_at(-4, input); + + const int64_t H_out = graph.size_at(-2, output); + const int64_t W_out = graph.size_at(-1, output); + const int64_t out_channels = graph.size_at(-3, output); + int64_t kernel_height; + int64_t kernel_width; + { + const auto kernel_size = graph.get_int_list(kernel_size_ref); + kernel_height = kernel_size->at(0); + kernel_width = kernel_size->at(1); + } + + const bool use_im2col = should_use_q8ta_conv2d_im2col({ + graph.device_is_mali(), + graph.can_use_int8_dot_product(), + static_cast(graph.max_buffer_numel()), + batch, + groups, + in_channels_per_group, + out_channels, + kernel_height, + kernel_width, + H_out, + W_out, + }); + + if (use_im2col) { + q8ta_conv2d_im2col(graph, args); + } else { + q8ta_conv2d_general_impl(graph, args); + } +} + +REGISTER_OPERATORS { + VK_REGISTER_OP(et_vk.q8ta_conv2d.default, q8ta_conv2d); +} + } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dRoute.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2d.h similarity index 84% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2dRoute.h rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2d.h index 9f60b6fd372..549ab2b25e8 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dRoute.h +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2d.h @@ -8,6 +8,8 @@ #pragma once +#include + #include namespace vkcompute { @@ -30,4 +32,6 @@ struct Q8taConv2dRouteParams final { bool should_use_q8ta_conv2d_im2col(const Q8taConv2dRouteParams& params); +void q8ta_conv2d(ComputeGraph& graph, const std::vector& args); + } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.cpp new file mode 100644 index 00000000000..bde1492ca18 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.cpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +namespace vkcompute { + +ActivationType activation_type_from_string(const std::string& activation) { + if (activation == "none") { + return ActivationType::kNone; + } else if (activation == "relu") { + return ActivationType::kRelu; + } + VK_THROW("Unknown activation type: ", activation); +} + +bool q8ta_conv2d_check_packed_dim_info(const api::PackedDimInfo& info) { + return info.packed_dim == WHCN::kChannelsDim && + info.packed_dim_block_size == 4 && + info.outer_packed_dim == WHCN::kWidthDim && + (info.outer_packed_dim_block_size == 1 || + info.outer_packed_dim_block_size == 4); +} + +bool q8ta_conv2d_check_4w4c_packed_dim_info(const api::PackedDimInfo& info) { + return info.packed_dim == WHCN::kChannelsDim && + info.packed_dim_block_size == 4 && + info.outer_packed_dim == WHCN::kWidthDim && + info.outer_packed_dim_block_size == 4; +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.h new file mode 100644 index 00000000000..5c9cb45d55e --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +#include + +namespace vkcompute { + +enum class ActivationType : uint32_t { + kNone = 0, + kRelu = 1, +}; + +ActivationType activation_type_from_string(const std::string& activation); + +bool q8ta_conv2d_check_packed_dim_info(const api::PackedDimInfo& info); + +bool q8ta_conv2d_check_4w4c_packed_dim_info(const api::PackedDimInfo& info); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dDW.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDW.cpp similarity index 97% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2dDW.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDW.cpp index a504d9ab3f6..b0696bb5442 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dDW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDW.cpp @@ -6,12 +6,14 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include + +#include #include #include -#include #include +#include #include #include @@ -437,7 +439,9 @@ void add_q8ta_conv2d_dw_node( // High level operator impl // -void q8ta_conv2d_dw(ComputeGraph& graph, const std::vector& args) { +void q8ta_conv2d_dw_impl( + ComputeGraph& graph, + const std::vector& args) { int32_t idx = 0; const ValueRef packed_int8_input = args.at(idx++); const ValueRef input_scale = args.at(idx++); @@ -509,7 +513,7 @@ void q8ta_conv2d_dw(ComputeGraph& graph, const std::vector& args) { } REGISTER_OPERATORS { - VK_REGISTER_OP(et_vk.q8ta_conv2d_dw.default, q8ta_conv2d_dw); + VK_REGISTER_OP(et_vk.q8ta_conv2d_dw.default, q8ta_conv2d_dw_impl); } } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDW.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDW.h new file mode 100644 index 00000000000..cbff43acfb9 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDW.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +namespace vkcompute { + +ValueRef prepack_quantized_conv2d_dw_weight( + ComputeGraph& graph, + const QuantizationConfig& weight_quant_config, + const ValueRef weight_data, + const ValueRef kernel_size); + +void add_q8ta_conv2d_dw_node( + ComputeGraph& graph, + const ValueRef packed_int8_input, + const ValueRef input_scale, + const ValueRef input_zp, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef output_scale, + const ValueRef output_zp, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef kernel_size, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef groups, + const uint32_t activation_type, + const ValueRef packed_int8_output); + +void add_conv2d_dw_q8ta_q8csw_q8to_4w4c_node( + ComputeGraph& graph, + const ValueRef packed_int8_input, + const ValueRef input_scale, + const ValueRef input_zp, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef output_scale, + const ValueRef output_zp, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef kernel_size, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef groups, + const ValueRef packed_int8_output); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDirect.cpp similarity index 84% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDirect.cpp index 75f65d7a2a0..d1a3edba2de 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDirect.cpp @@ -6,43 +6,19 @@ * LICENSE file in the root directory of this source tree. */ -#include -#include +#include #include #include -#include #include +#include +#include #include #include namespace vkcompute { -ActivationType activation_type_from_string(const std::string& activation) { - if (activation == "none") { - return ActivationType::kNone; - } else if (activation == "relu") { - return ActivationType::kRelu; - } - VK_THROW("Unknown activation type: ", activation); -} - -bool q8ta_conv2d_check_packed_dim_info(const api::PackedDimInfo& info) { - return info.packed_dim == WHCN::kChannelsDim && - info.packed_dim_block_size == 4 && - info.outer_packed_dim == WHCN::kWidthDim && - (info.outer_packed_dim_block_size == 1 || - info.outer_packed_dim_block_size == 4); -} - -bool q8ta_conv2d_check_4w4c_packed_dim_info(const api::PackedDimInfo& info) { - return info.packed_dim == WHCN::kChannelsDim && - info.packed_dim_block_size == 4 && - info.outer_packed_dim == WHCN::kWidthDim && - info.outer_packed_dim_block_size == 4; -} - // // Workgroup size selection functions // @@ -160,7 +136,8 @@ ValueRef prepack_quantized_conv2d_weight( VK_CHECK_COND(weight_quant_config.nbits == 8); VK_CHECK_COND(weight_quant_config.is_symmetric); - const int32_t groups_val = graph.get_int(groups); + const int32_t groups_val = + utils::safe_downcast(graph.get_int(groups)); const int64_t OC = graph.size_at(-3, output); const int64_t IC = graph.size_at(-3, input) / groups_val; @@ -397,7 +374,7 @@ void add_q8ta_conv2d_node( // High level operator impl // -void q8ta_conv2d_general( +void q8ta_conv2d_general_impl( ComputeGraph& graph, const std::vector& args) { int32_t idx = 0; @@ -479,56 +456,8 @@ void q8ta_conv2d_general( packed_int8_output); } -void q8ta_conv2d(ComputeGraph& graph, const std::vector& args) { - const ValueRef input = args.at(0); - const ValueRef kernel_size_ref = args.at(9); - const ValueRef groups_ref = args.at(13); - const ValueRef output = args.at(15); - - const int64_t groups = graph.extract_scalar(groups_ref); - // Valid models always carry groups >= 1; fail fast on corrupt input - // instead of dividing channel counts by zero downstream (both this - // dispatcher and q8ta_conv2d_general divide by groups). - VK_CHECK_COND(groups > 0, "q8ta_conv2d requires groups >= 1"); - const int64_t in_channels = graph.size_at(-3, input); - const int64_t in_channels_per_group = in_channels / groups; - const int64_t batch = graph.size_at(-4, input); - - const int64_t H_out = graph.size_at(-2, output); - const int64_t W_out = graph.size_at(-1, output); - const int64_t out_channels = graph.size_at(-3, output); - int64_t kernel_height; - int64_t kernel_width; - { - const auto kernel_size = graph.get_int_list(kernel_size_ref); - kernel_height = kernel_size->at(0); - kernel_width = kernel_size->at(1); - } - - const bool use_im2col = should_use_q8ta_conv2d_im2col({ - graph.device_is_mali(), - graph.can_use_int8_dot_product(), - static_cast(graph.max_buffer_numel()), - batch, - groups, - in_channels_per_group, - out_channels, - kernel_height, - kernel_width, - H_out, - W_out, - }); - - if (use_im2col) { - q8ta_conv2d_im2col(graph, args); - } else { - q8ta_conv2d_general(graph, args); - } -} - REGISTER_OPERATORS { - VK_REGISTER_OP(et_vk.q8ta_conv2d.default, q8ta_conv2d); - VK_REGISTER_OP(et_vk.q8ta_conv2d_general.default, q8ta_conv2d_general); + VK_REGISTER_OP(et_vk.q8ta_conv2d_general.default, q8ta_conv2d_general_impl); } } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDirect.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDirect.h new file mode 100644 index 00000000000..e1960303e1f --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDirect.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +namespace vkcompute { + +ValueRef prepack_quantized_conv2d_weight( + ComputeGraph& graph, + const QuantizationConfig& weight_quant_config, + const ValueRef weight_data, + const ValueRef input, + const ValueRef output, + const ValueRef groups, + const ValueRef kernel_size); + +void add_q8ta_conv2d_node( + ComputeGraph& graph, + const ValueRef packed_int8_input, + const ValueRef packed_int8_input_im2col, + const ValueRef input_scale, + const ValueRef input_zp, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef output_scale, + const ValueRef output_zp, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef kernel_size, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef groups, + const uint32_t activation_type, + const ValueRef packed_int8_output); + +void q8ta_conv2d_general_impl( + ComputeGraph& graph, + const std::vector& args); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dPW.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dPW.cpp similarity index 98% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2dPW.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dPW.cpp index 454a115498b..d4bb4dbab38 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dPW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dPW.cpp @@ -6,10 +6,13 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include + +#include #include #include +#include #include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dPW.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dPW.h new file mode 100644 index 00000000000..97878ee122c --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dPW.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +void add_q8ta_conv2d_pw_node( + ComputeGraph& graph, + const bool use_unsigned_dot, + const ValueRef packed_int8_input, + const ValueRef input_scale, + const ValueRef input_zp, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef output_scale, + const ValueRef output_zp, + const ValueRef bias_data, + const ValueRef packed_bias, + const uint32_t activation_type, + const ValueRef packed_int8_output, + const int32_t groups = 1, + const ValueRef conv_input = kDummyValueRef, + const ValueRef kernel_size = kDummyValueRef, + const ValueRef stride = kDummyValueRef, + const ValueRef padding = kDummyValueRef, + const ValueRef dilation = kDummyValueRef, + const bool is_im2col = false, + const ValueRef stream_row_offset_ref = kDummyValueRef, + const ValueRef max_im2col_rows_ref = kDummyValueRef); + +constexpr int64_t kMaxUnsignedDotAccumulatorBytes = 33025; + +bool can_use_unsigned_pw_dot( + const vkapi::Adapter& adapter, + int64_t k_per_group); + +void q8ta_conv2d_pw_impl( + ComputeGraph& graph, + bool use_unsigned_dot, + const std::vector& args); + +void q8ta_conv2d_pw(ComputeGraph& graph, const std::vector& args); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dIm2Col.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.cpp similarity index 96% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2dIm2Col.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.cpp index 062e5ea1d8d..39f9a44b476 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dIm2Col.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.cpp @@ -6,15 +6,18 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include + +#include +#include +#include #include #include -#include #include #include +#include #include -#include #include #include @@ -372,7 +375,7 @@ void q8ta_conv2d_im2col_impl( full_im2col_sizes.at(3), graph.max_buffer_numel()); if (!stream_plan.feasible) { - q8ta_conv2d_general(graph, args); + q8ta_conv2d_general_impl(graph, args); return; } VK_CHECK_COND( diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.h new file mode 100644 index 00000000000..c1520726111 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +inline constexpr int64_t kQ8taConv2dIm2ColScratchBudgetBytes = 16 * 1024 * 1024; +inline constexpr int64_t kQ8taConv2dMaxRowsPerTile = 65535; + +struct Q8taConv2dStreamPlan final { + int64_t aligned_out_width; + int64_t rows_per_tile; + int64_t num_tiles; + int64_t scratch_bytes; + bool feasible; +}; + +Q8taConv2dStreamPlan make_q8ta_conv2d_stream_plan( + int64_t batch, + int64_t flattened_kernel_size, + int64_t out_height, + int64_t out_width, + int64_t scratch_budget_bytes); + +// max_buffer_bytes is Adapter::max_buffer_numel(), which returns +// maxStorageBufferRange in bytes (not elements) — directly comparable with +// the byte-denominated scratch budget. +Q8taConv2dStreamPlan make_q8ta_conv2d_stream_plan_for_device( + int64_t batch, + int64_t flattened_kernel_size, + int64_t out_height, + int64_t out_width, + uint64_t max_buffer_bytes); + +std::vector calculate_q8ta_im2col_sizes( + ComputeGraph* graph, + const ValueRef& input, + const ValueRef& output, + const ValueRef& kernel_size, + const ValueRef& groups); + +void add_q8ta_im2col_node( + ComputeGraph& graph, + const ValueRef packed_int8_input, + const ValueRef kernel_size, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef groups, + const ValueRef packed_int8_output, + const ValueRef packed_int8_im2col, + const int32_t zp, + const ValueRef stream_row_offset_ref, + const ValueRef max_im2col_rows_ref = kDummyValueRef); + +void q8ta_conv2d_im2col(ComputeGraph& graph, const std::vector& args); + +void q8ta_conv2d_im2col_impl( + ComputeGraph& graph, + bool use_unsigned_dot, + const std::vector& args); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dTransposed.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/transposed/Q8taConv2dTransposed.cpp similarity index 95% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2dTransposed.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/transposed/Q8taConv2dTransposed.cpp index 91d745fb1fa..ff5cac10a0b 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dTransposed.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/transposed/Q8taConv2dTransposed.cpp @@ -6,13 +6,16 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include + +#include +#include #include #include -#include #include +#include #include #include @@ -228,7 +231,7 @@ void add_q8ta_conv2d_transposed_node( resize_q8ta_conv2d_transposed_node)); } -void q8ta_conv2d_transposed( +void q8ta_conv2d_transposed_impl( ComputeGraph& graph, const std::vector& args) { int32_t idx = 0; @@ -313,7 +316,8 @@ void q8ta_conv2d_transposed( } REGISTER_OPERATORS { - VK_REGISTER_OP(et_vk.q8ta_conv2d_transposed.default, q8ta_conv2d_transposed); + VK_REGISTER_OP( + et_vk.q8ta_conv2d_transposed.default, q8ta_conv2d_transposed_impl); } } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/transposed/Q8taConv2dTransposed.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/transposed/Q8taConv2dTransposed.h new file mode 100644 index 00000000000..92607284b00 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/transposed/Q8taConv2dTransposed.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +void q8ta_conv2d_transposed_impl( + ComputeGraph& graph, + const std::vector& args); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedConvolution.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.cpp similarity index 98% rename from backends/vulkan/runtime/graph/ops/impl/QuantizedConvolution.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.cpp index 30415cd4f08..595395b6f82 100644 --- a/backends/vulkan/runtime/graph/ops/impl/QuantizedConvolution.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.cpp @@ -9,13 +9,15 @@ #include #include -#include -#include #include #include -#include #include #include +#include +#include +#include +#include +#include #include namespace vkcompute { @@ -1341,7 +1343,7 @@ void conv2d_q8ta_q8csw_q8to( // Test operators // -void test_conv2d_q8ta_q8csw_q8to( +void test_conv2d_q8ta_q8csw_q8to_impl( ComputeGraph& graph, const std::vector& args) { int32_t idx = 0; @@ -1407,7 +1409,8 @@ REGISTER_OPERATORS { VK_REGISTER_OP(et_vk.conv2d_q8ta_q8csw.default, conv2d_q8ta_q8csw); VK_REGISTER_OP(et_vk.conv2d_q8csw.default, conv2d_q8csw); VK_REGISTER_OP( - etvk.test_conv2d_q8ta_q8csw_q8to.default, test_conv2d_q8ta_q8csw_q8to); + etvk.test_conv2d_q8ta_q8csw_q8to.default, + test_conv2d_q8ta_q8csw_q8to_impl); VK_REGISTER_OP(et_vk.conv2d_q8ta_q8csw_q8to.default, conv2d_q8ta_q8csw_q8to); VK_REGISTER_OP( et_vk.conv2d_q8ta_q8csw_q8to_dw.default, conv2d_q8ta_q8csw_q8to); diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedConvolution.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.h similarity index 78% rename from backends/vulkan/runtime/graph/ops/impl/QuantizedConvolution.h rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.h index 8913e639f0e..6bc453d0d27 100644 --- a/backends/vulkan/runtime/graph/ops/impl/QuantizedConvolution.h +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.h @@ -8,7 +8,7 @@ #pragma once -#include +#include namespace vkcompute { diff --git a/backends/vulkan/test/custom_ops/impl/TestConv2d.cpp b/backends/vulkan/test/custom_ops/impl/TestConv2d.cpp index 343f416c6a1..bad30507ac1 100644 --- a/backends/vulkan/test/custom_ops/impl/TestConv2d.cpp +++ b/backends/vulkan/test/custom_ops/impl/TestConv2d.cpp @@ -9,8 +9,8 @@ #include #include -#include -#include +#include +#include #include @@ -32,8 +32,8 @@ void test_conv2d(ComputeGraph& graph, const std::vector& args) { // impl_selector grammar: // "" -> aten.convolution.default (heuristic-routed: // should_use_conv2d_im2col() picks direct vs im2col) - // "direct" -> add_conv2d_node(force_direct=true): forces the direct - // sliding-window path, bypassing the routing heuristic + // "direct" -> conv2d_direct_impl: forces the direct sliding-window + // path, bypassing the routing heuristic // "im2col" -> et_vk.conv2d_gemm.default, auto im2col storage // "im2col_buffer"-> im2col/GEMM, force buffer im2col intermediate // "im2col_tex2d" -> im2col/GEMM, force texture2d im2col intermediate @@ -94,11 +94,10 @@ void test_conv2d(ComputeGraph& graph, const std::vector& args) { // The "direct" selector must reach the exact direct sliding-window dispatch // the heuristic would otherwise pick. The registered op can only route via - // the heuristic, so call add_conv2d_node directly with force_direct=true to - // bypass it (mirroring how the forced-storage variants call - // conv2d_gemm_impl). + // the heuristic, so call conv2d_direct_impl directly to bypass it (mirroring + // how the forced-storage variants call conv2d_gemm_impl). if (impl_selector == "direct") { - add_conv2d_node( + conv2d_direct_impl( graph, input, weight, @@ -109,11 +108,10 @@ void test_conv2d(ComputeGraph& graph, const std::vector& args) { transposed, output_padding, groups, - /*out_min=*/kDummyValueRef, - /*out_max=*/kDummyValueRef, out, /*clamp_out=*/false, - /*force_direct=*/true); + /*out_min_val=*/0.0f, + /*out_max_val=*/0.0f); return; } diff --git a/backends/vulkan/test/custom_ops/impl/TestConv2dDw.cpp b/backends/vulkan/test/custom_ops/impl/TestConv2dDw.cpp index 85c2308b11f..f27a596eca3 100644 --- a/backends/vulkan/test/custom_ops/impl/TestConv2dDw.cpp +++ b/backends/vulkan/test/custom_ops/impl/TestConv2dDw.cpp @@ -9,8 +9,9 @@ #include #include -#include #include +#include +#include #include #include diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taConv2d.cpp b/backends/vulkan/test/custom_ops/impl/TestQ8taConv2d.cpp index e4af2c56b9d..4da454040e7 100644 --- a/backends/vulkan/test/custom_ops/impl/TestQ8taConv2d.cpp +++ b/backends/vulkan/test/custom_ops/impl/TestQ8taConv2d.cpp @@ -10,8 +10,9 @@ #include #include -#include #include +#include +#include namespace vkcompute { diff --git a/backends/vulkan/test/custom_ops/q8ta_conv2d_route_test.cpp b/backends/vulkan/test/custom_ops/q8ta_conv2d_route_test.cpp index 63d154b170f..c660430b579 100644 --- a/backends/vulkan/test/custom_ops/q8ta_conv2d_route_test.cpp +++ b/backends/vulkan/test/custom_ops/q8ta_conv2d_route_test.cpp @@ -4,7 +4,7 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include +#include #include diff --git a/backends/vulkan/test/custom_ops/q8ta_conv2d_stream_plan_test.cpp b/backends/vulkan/test/custom_ops/q8ta_conv2d_stream_plan_test.cpp index 6f634a83c27..c1207cb1923 100644 --- a/backends/vulkan/test/custom_ops/q8ta_conv2d_stream_plan_test.cpp +++ b/backends/vulkan/test/custom_ops/q8ta_conv2d_stream_plan_test.cpp @@ -4,7 +4,7 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include +#include #include diff --git a/backends/vulkan/test/custom_ops/test_conv2d.cpp b/backends/vulkan/test/custom_ops/test_conv2d.cpp index 6bec62a4e61..2f21f0532fe 100644 --- a/backends/vulkan/test/custom_ops/test_conv2d.cpp +++ b/backends/vulkan/test/custom_ops/test_conv2d.cpp @@ -526,7 +526,7 @@ static std::vector generate_conv2d_test_cases() { // "" -> heuristic-routed (should_use_conv2d_im2col picks direct on // Adreno for small c_out, im2col on Mali) // "im2col" -> forced im2col/GEMM path - // "direct" -> forced direct sliding-window path (force_direct=true) + // "direct" -> forced direct sliding-window path (conv2d_direct_impl) // Including "direct" guarantees the direct shader gets reference-checked on // BOTH devices — without it, Mali would always route "" to im2col and never // exercise the direct path. diff --git a/backends/vulkan/test/custom_ops/test_q8ta_conv2d.cpp b/backends/vulkan/test/custom_ops/test_q8ta_conv2d.cpp index 90fd4e1616b..17045225a2c 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_conv2d.cpp +++ b/backends/vulkan/test/custom_ops/test_q8ta_conv2d.cpp @@ -12,7 +12,8 @@ #include #include -#include +#include +#include #include #include diff --git a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_pw.cpp b/backends/vulkan/test/custom_ops/test_q8ta_conv2d_pw.cpp index 1003c0540ce..5a773f43319 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_pw.cpp +++ b/backends/vulkan/test/custom_ops/test_q8ta_conv2d_pw.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include diff --git a/backends/vulkan/test/op_tests/conv2d_gemm_dynamic_test.cpp b/backends/vulkan/test/op_tests/conv2d_gemm_dynamic_test.cpp index e5a7216bfe7..f993c453fb0 100644 --- a/backends/vulkan/test/op_tests/conv2d_gemm_dynamic_test.cpp +++ b/backends/vulkan/test/op_tests/conv2d_gemm_dynamic_test.cpp @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include From 0ff3de88ff2564e794962e8893edeccd5f5d7beb Mon Sep 17 00:00:00 2001 From: Stephen Jia Date: Mon, 12 Oct 2026 12:45:47 -0700 Subject: [PATCH 5/7] [ET-VK][gemm] Nest linear and matmul sources by scheme Pull Request resolved: https://github.com/pytorch/executorch/pull/23493 ## Ulterior Motive Organize Vulkan operators by implementation family so operators with multiple routes can use clear, scalable directory layouts. ## Rationale **What**: Move the linear and matrix multiplication C++ implementations, GLSL templates, YAML variant definitions, and GEMM tile headers under `gemm/` in both `impl/` and `glsl/`, split into `linear/` (an activation times a usually prepacked weight) and `matmul/` (multiplication between live activations). Quantized linear sources are grouped by how they are quantized, and the multi-scheme `QuantizedLinear.cpp` and the shared `GemmCoopmat.cpp` are split accordingly. **Why**: Linear and matmul sources were interleaved by name in the flat directories, quantized linear variants for different schemes shared one file, and the shared GEMM tile headers were scattered across the shader root, which obscured which sources implement which operator, route, and scheme. ## Details - Layout (`impl/` and `glsl/` mirror each other) - `gemm/`: `GemmCommon.{h,cpp}` (fp weight prepack, resize helpers, and the cooperative-matrix tile constants and `is_coopmat_eligible()` shared by linear and matmul) and `coopmat_mm`, the cooperative-matrix shader shared by both. - `gemm/tile_utils/`: all GEMM tile headers (tile structs, tile load/store/compute, block packing, accumulators, and scale/sum/bias loaders), also used by convolution, SDPA, and quantize shaders. - `gemm/linear/`: `Linear.{h,cpp}` (`aten.linear`), `LinearCoopmat.{h,cpp}`, and `backward/LinearDw.cpp` (`et_vk.linear_dW`). - `gemm/linear/q8ta/`: `Q8taLinear` and `Q8taLinearGemv`; activations and weights are quantized ahead of time with static parameters (primarily statically quantized convolutional models). - `gemm/linear/qw/`: weight-only quantization, with activations kept in floating point or quantized dynamically (primarily LLMs and transformers). `QuantizedLinear.{h,cpp}` hold the shared weight prepacking, workgroup sizing, the weight-only node, and `quantized_linear_impl`; schemes live in `q4gsw/` (with `backward/` for `et_vk.linear_q4gsw_backward` and `et_vk.q4gsw_requant`), `dq8ca_q4gsw/`, `q8csw/`, and `legacy/` (`QuantizedLinearQCSNW` and the `q8ta_q8csw` implementation that runs through `quantized_linear_impl`). - `gemm/matmul/`: `MatMul.{h,cpp}` (`aten.mm`, `aten.bmm`, `aten.addmm`; renamed from `Matmul.cpp` to match its header) and `MatmulCoopmat.{h,cpp}`. - Weight packing shaders sit in `pack_weights/` subdirectories of the folder that dispatches them (`linear/`, `linear/qw/`, `linear/qw/q4gsw/`, and the shader root for `pack_int4_linear_weight_transposed_interleaved`). - Splits - `QuantizedLinear.cpp`: `q8csw/Q8cswLinear.cpp` (`et_vk.linear_q8csw`, `aten._weight_int8pack_mm`), `legacy/Q8taQ8cswLinear.{h,cpp}` (`add_linear_qa_qw_node`, `et_vk.linear_q8ta_q8csw`), and `dq8ca_q4gsw/Dq8caQ4gswLinear.{h,cpp}` (`add_linear_dqa_qw_node`, `et_vk.linear_dq8ca_q4gsw`). `can_use_q4gsw_coopmat` and the coopmat tile constants are now declared in `QuantizedLinear.h` because the shared and `dq8ca_q4gsw` files both use them. - `GemmCoopmat.cpp`: `linear/LinearCoopmat.cpp` and `matmul/MatmulCoopmat.cpp`. - Functions move unchanged. - Includes - Shaders inside `gemm/` include tile headers as `tile_utils/
`; shaders outside `gemm/` include them as `gemm/tile_utils/
`. - C++ include paths and `op_registry.py` section labels are updated. - Moved files are recorded as renames, and split files as copies of their source. - Code that moves within or between files is re-linted, so its CLANGTIDY findings are fixed: `(void)` for unused parameters, `utils::safe_downcast` for narrowing in `quantized_linear_gwg` and the coopmat size reads, bounds-checked `.at()` in place of `.back()` in `LinearCoopmat.cpp`, and removal of unused includes (`Softmax.h` in `SDPA.cpp`, `Linear.h` in `Conv1dPW.cpp`, and two utility headers in the coopmat files). Generated shader names and runtime dispatch behavior are unchanged. ghstack-source-id: 446951723 @exported-using-ghexport Differential Revision: [D123523650](https://our.internmc.facebook.com/intern/diff/D123523650/) --- backends/vulkan/op_registry.py | 10 +- .../ops/glsl/conv2d_fp_input_tile_load.glslh | 2 +- .../glsl/convolution/conv1d/conv1d_pw.glsl | 10 +- .../convolution/conv2d/conv2d_pw_tiled.glsl | 6 +- .../conv2d/im2col/conv2d_gemm.glsl | 6 +- .../conv2d_fp_im2col_block_load.glslh | 2 +- .../conv2d_fp_im2col_block_store.glslh | 2 +- .../conv2d_int8_input_tile_load.glslh | 2 +- .../conv2d_int8_output_tile_store.glslh | 2 +- .../conv2d_pw_q8ta_q8csw_q8to_tiled.glsl | 12 +- .../quantized_legacy/conv2d_q8_utils.glslh | 2 +- .../conv2d_q8csw_linear_tiled.glsl | 12 +- .../conv2d_q8ta_q8csw_linear_tiled.glsl | 12 +- .../conv2d_q8ta_q8csw_q8to.glsl | 12 +- .../conv2d_q8ta_q8csw_q8to_linear_tiled.glsl | 12 +- .../quantize_and_pack_im2col.glsl | 2 +- .../graph/ops/glsl/{ => gemm}/coopmat_mm.glsl | 0 .../graph/ops/glsl/{ => gemm}/coopmat_mm.yaml | 0 .../{ => gemm/linear/backward}/linear_dW.glsl | 0 .../{ => gemm/linear/backward}/linear_dW.yaml | 0 .../glsl/{ => gemm/linear}/linear_scalar.glsl | 10 +- .../glsl/{ => gemm/linear}/linear_scalar.yaml | 0 .../glsl/{ => gemm/linear}/linear_vec.glsl | 10 +- .../glsl/{ => gemm/linear}/linear_vec.yaml | 0 .../pack_weights}/pack_fp_linear_weight.glsl | 0 .../pack_weights}/pack_fp_linear_weight.yaml | 0 .../{ => gemm/linear/q8ta}/q8ta_linear.glsl | 12 +- .../{ => gemm/linear/q8ta}/q8ta_linear.yaml | 0 .../linear/q8ta}/q8ta_linear_gemv.glsl | 10 +- .../linear/q8ta}/q8ta_linear_gemv.yaml | 0 .../linear_dq8ca_q4gsw_tiled.glsl | 26 +- .../linear_dq8ca_q4gsw_tiled.yaml | 0 .../dq8ca_q4gsw}/linear_dq8ca_qw_coopmat.glsl | 0 .../dq8ca_q4gsw}/linear_dq8ca_qw_coopmat.yaml | 0 .../qw/legacy}/linear_q8ta_q8csw_tiled.glsl | 14 +- .../qw/legacy}/linear_q8ta_q8csw_tiled.yaml | 0 .../linear/qw/legacy}/linear_qcsnw.glsl | 0 .../linear/qw/legacy}/linear_qcsnw.yaml | 0 .../linear/qw/legacy}/linear_qcsnw_coop.glsl | 0 .../linear/qw/legacy}/linear_qcsnw_coop.yaml | 0 .../linear/qw/legacy}/linear_qcsnw_tiled.glsl | 0 .../linear/qw/legacy}/linear_qcsnw_tiled.yaml | 0 .../pack_weights}/pack_q8_linear_weight.glsl | 2 +- .../pack_weights}/pack_q8_linear_weight.yaml | 0 .../qw/q4gsw/backward}/q4gsw_backward.glsl | 0 .../qw/q4gsw/backward}/q4gsw_backward.yaml | 0 .../qw/q4gsw/backward}/q4gsw_requant.glsl | 0 .../qw/q4gsw/backward}/q4gsw_requant.yaml | 0 .../linear/qw/q4gsw}/linear_q4gsw_coop.glsl | 14 +- .../linear/qw/q4gsw}/linear_q4gsw_coop.yaml | 0 .../linear/qw/q4gsw}/linear_q4gsw_tiled.glsl | 14 +- .../linear/qw/q4gsw}/linear_q4gsw_tiled.yaml | 0 .../linear/qw/q4gsw}/linear_qw_coopmat.glsl | 0 .../linear/qw/q4gsw}/linear_qw_coopmat.yaml | 0 .../pack_weights}/pack_q4_linear_weight.glsl | 2 +- .../pack_weights}/pack_q4_linear_weight.yaml | 0 .../pack_q4_linear_weight__w_4x8.glsl | 0 .../pack_q4_linear_weight__w_4x8.yaml | 0 .../q4gsw}/q4gsw_linear_gemm__tin__w_4x8.glsl | 0 .../q4gsw}/q4gsw_linear_gemm__tin__w_4x8.yaml | 0 .../qw/q4gsw}/q4gsw_linear_gemm__w_4x8.glsl | 0 .../qw/q4gsw}/q4gsw_linear_gemm__w_4x8.yaml | 0 .../q4gsw}/q4gsw_linear_gemv_coop__w_4x8.glsl | 0 .../q4gsw}/q4gsw_linear_gemv_coop__w_4x8.yaml | 0 .../linear/qw/q8csw}/linear_q8csw_tiled.glsl | 16 +- .../linear/qw/q8csw}/linear_q8csw_tiled.yaml | 0 .../glsl/{ => gemm/matmul}/matmul_scalar.glsl | 10 +- .../glsl/{ => gemm/matmul}/matmul_scalar.yaml | 0 .../glsl/{ => gemm/matmul}/matmul_vec.glsl | 10 +- .../glsl/{ => gemm/matmul}/matmul_vec.yaml | 0 .../{ => gemm/tile_utils}/linear_common.glslh | 0 .../tile_utils}/linear_fp_bias_load.glslh | 0 .../tile_utils}/linear_fp_input_tile.glslh | 0 .../linear_fp_input_tile_load.glslh | 0 .../tile_utils}/linear_fp_output_tile.glslh | 0 .../linear_fp_output_tile_fp_compute.glslh | 0 ...inear_fp_output_tile_fp_int4_compute.glslh | 0 ...inear_fp_output_tile_fp_int8_compute.glslh | 0 ...ear_fp_output_tile_int8_int4_compute.glslh | 0 ...ear_fp_output_tile_int8_int8_compute.glslh | 0 .../linear_fp_output_tile_store.glslh | 0 .../linear_fp_packed_weight_tile_load.glslh | 0 .../linear_fp_per_out_channel_params.glslh | 0 .../linear_fp_weight_scales_load.glslh | 0 .../tile_utils}/linear_fp_weight_tile.glslh | 0 .../linear_int4_weight_block.glslh | 0 .../tile_utils}/linear_int4_weight_tile.glslh | 0 .../linear_int4_weight_tile_load.glslh | 0 .../tile_utils}/linear_int8_input_block.glslh | 0 .../linear_int8_input_scales_zps.glslh | 0 .../linear_int8_input_scales_zps_load.glslh | 0 .../linear_int8_input_sums_load.glslh | 0 .../linear_int8_input_sums_store.glslh | 0 .../tile_utils}/linear_int8_input_tile.glslh | 0 .../linear_int8_input_tile_load.glslh | 0 .../tile_utils}/linear_int8_output_tile.glslh | 0 .../linear_int8_output_tile_compute.glslh | 0 .../linear_int8_weight_block.glslh | 0 .../tile_utils}/linear_int8_weight_tile.glslh | 0 .../linear_int8_weight_tile_load.glslh | 0 .../tile_utils}/linear_int_accumulator.glslh | 0 .../linear_int_per_in_channel_params.glslh | 0 .../linear_int_per_out_channel_params.glslh | 0 .../linear_int_weight_sums_load.glslh | 0 .../tile_utils}/matmul_fp_bias_apply.glslh | 0 .../matmul_fp_mat1_tile_load.glslh | 0 .../matmul_fp_mat2_tile_load.glslh | 0 .../matmul_fp_out_tile_store.glslh | 0 ..._linear_weight_transposed_interleaved.glsl | 0 ..._linear_weight_transposed_interleaved.yaml | 0 .../ops/glsl/quantize_and_pack_4h4w.glsl | 4 +- ...uantize_and_pack_4h4w_with_group_sums.glsl | 6 +- .../ops/glsl/quantize_and_pack_4w4c.glsl | 2 +- .../glsl/sdpa_compute_attn_weights_coop.glsl | 2 +- .../glsl/sdpa_compute_attn_weights_tiled.glsl | 2 +- .../graph/ops/glsl/sdpa_compute_out_coop.glsl | 2 +- .../ops/glsl/sdpa_compute_out_tiled.glsl | 2 +- .../glsl/sdpa_fp_attn_weight_tile_load.glslh | 2 +- .../glsl/sdpa_fp_attn_weight_tile_store.glslh | 2 +- .../ops/glsl/sdpa_fp_k_cache_tile_load.glslh | 2 +- .../ops/glsl/sdpa_fp_out_tile_store.glslh | 2 +- .../glsl/sdpa_fp_q_projected_tile_load.glslh | 2 +- .../ops/glsl/sdpa_fp_v_cache_tile_load.glslh | 2 +- .../ops/glsl/unpack_4w4c_and_dequantize.glsl | 4 +- .../runtime/graph/ops/impl/EmbeddingQ4gsw.cpp | 2 +- .../runtime/graph/ops/impl/GemmCommon.h | 45 -- .../runtime/graph/ops/impl/QuantizedLinear.h | 30 -- .../vulkan/runtime/graph/ops/impl/SDPA.cpp | 3 +- .../ops/impl/convolution/conv1d/Conv1dPW.cpp | 1 - .../conv2d/q8ta/im2col/Q8taConv2dIm2Col.cpp | 2 +- .../quantized_legacy/QuantizedConvolution.cpp | 2 +- .../graph/ops/impl/{ => gemm}/GemmCommon.cpp | 2 +- .../impl/{GemmCoopmat.h => gemm/GemmCommon.h} | 50 ++- .../ops/impl/{ => gemm/linear}/Linear.cpp | 8 +- .../graph/ops/impl/{ => gemm/linear}/Linear.h | 0 .../linear/LinearCoopmat.cpp} | 92 +---- .../ops/impl/gemm/linear/LinearCoopmat.h | 24 ++ .../{ => gemm/linear/backward}/LinearDw.cpp | 0 .../{ => gemm/linear/q8ta}/Q8taLinear.cpp | 4 +- .../impl/{ => gemm/linear/q8ta}/Q8taLinear.h | 0 .../{ => gemm/linear/q8ta}/Q8taLinearGemv.cpp | 4 +- .../{ => gemm/linear/q8ta}/Q8taLinearGemv.h | 0 .../{ => gemm/linear/qw}/QuantizedLinear.cpp | 386 +----------------- .../ops/impl/gemm/linear/qw/QuantizedLinear.h | 87 ++++ .../qw/dq8ca_q4gsw/Dq8caQ4gswLinear.cpp | 192 +++++++++ .../linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.h | 36 ++ .../gemm/linear/qw/legacy/Q8taQ8cswLinear.cpp | 128 ++++++ .../gemm/linear/qw/legacy/Q8taQ8cswLinear.h | 35 ++ .../qw/legacy}/QuantizedLinearQCSNW.cpp | 2 +- .../linear/qw/q4gsw}/Q4gswLinear.cpp | 2 +- .../{ => gemm/linear/qw/q4gsw}/Q4gswLinear.h | 0 .../backward}/QuantizedLinearBackward.cpp | 2 +- .../backward}/QuantizedLinearRequant.cpp | 0 .../impl/gemm/linear/qw/q8csw/Q8cswLinear.cpp | 83 ++++ .../{Matmul.cpp => gemm/matmul/MatMul.cpp} | 9 +- .../graph/ops/impl/{ => gemm/matmul}/MatMul.h | 0 .../ops/impl/gemm/matmul/MatmulCoopmat.cpp | 93 +++++ .../ops/impl/gemm/matmul/MatmulCoopmat.h | 21 + .../custom_ops/impl/TestFpaQ4gswLinear.cpp | 2 +- .../test/custom_ops/impl/TestMatmulLinear.cpp | 9 +- 160 files changed, 926 insertions(+), 722 deletions(-) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm}/coopmat_mm.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm}/coopmat_mm.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/backward}/linear_dW.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/backward}/linear_dW.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear}/linear_scalar.glsl (90%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear}/linear_scalar.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear}/linear_vec.glsl (90%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear}/linear_vec.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/pack_weights}/pack_fp_linear_weight.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/pack_weights}/pack_fp_linear_weight.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/q8ta}/q8ta_linear.glsl (92%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/q8ta}/q8ta_linear.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/q8ta}/q8ta_linear_gemv.glsl (94%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/q8ta}/q8ta_linear_gemv.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/dq8ca_q4gsw}/linear_dq8ca_q4gsw_tiled.glsl (85%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/dq8ca_q4gsw}/linear_dq8ca_q4gsw_tiled.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/dq8ca_q4gsw}/linear_dq8ca_qw_coopmat.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/dq8ca_q4gsw}/linear_dq8ca_qw_coopmat.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/legacy}/linear_q8ta_q8csw_tiled.glsl (89%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/legacy}/linear_q8ta_q8csw_tiled.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/legacy}/linear_qcsnw.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/legacy}/linear_qcsnw.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/legacy}/linear_qcsnw_coop.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/legacy}/linear_qcsnw_coop.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/legacy}/linear_qcsnw_tiled.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/legacy}/linear_qcsnw_tiled.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/pack_weights}/pack_q8_linear_weight.glsl (97%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/pack_weights}/pack_q8_linear_weight.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw/backward}/q4gsw_backward.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw/backward}/q4gsw_backward.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw/backward}/q4gsw_requant.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw/backward}/q4gsw_requant.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw}/linear_q4gsw_coop.glsl (92%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw}/linear_q4gsw_coop.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw}/linear_q4gsw_tiled.glsl (89%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw}/linear_q4gsw_tiled.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw}/linear_qw_coopmat.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw}/linear_qw_coopmat.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw/pack_weights}/pack_q4_linear_weight.glsl (97%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw/pack_weights}/pack_q4_linear_weight.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw/pack_weights}/pack_q4_linear_weight__w_4x8.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw/pack_weights}/pack_q4_linear_weight__w_4x8.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw}/q4gsw_linear_gemm__tin__w_4x8.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw}/q4gsw_linear_gemm__tin__w_4x8.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw}/q4gsw_linear_gemm__w_4x8.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw}/q4gsw_linear_gemm__w_4x8.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw}/q4gsw_linear_gemv_coop__w_4x8.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q4gsw}/q4gsw_linear_gemv_coop__w_4x8.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q8csw}/linear_q8csw_tiled.glsl (87%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/linear/qw/q8csw}/linear_q8csw_tiled.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/matmul}/matmul_scalar.glsl (90%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/matmul}/matmul_scalar.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/matmul}/matmul_vec.glsl (90%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/matmul}/matmul_vec.yaml (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_common.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_bias_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_input_tile.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_input_tile_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_output_tile.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_output_tile_fp_compute.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_output_tile_fp_int4_compute.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_output_tile_fp_int8_compute.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_output_tile_int8_int4_compute.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_output_tile_int8_int8_compute.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_output_tile_store.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_packed_weight_tile_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_per_out_channel_params.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_weight_scales_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_fp_weight_tile.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int4_weight_block.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int4_weight_tile.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int4_weight_tile_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int8_input_block.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int8_input_scales_zps.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int8_input_scales_zps_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int8_input_sums_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int8_input_sums_store.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int8_input_tile.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int8_input_tile_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int8_output_tile.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int8_output_tile_compute.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int8_weight_block.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int8_weight_tile.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int8_weight_tile_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int_accumulator.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int_per_in_channel_params.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int_per_out_channel_params.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/linear_int_weight_sums_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/matmul_fp_bias_apply.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/matmul_fp_mat1_tile_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/matmul_fp_mat2_tile_load.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => gemm/tile_utils}/matmul_fp_out_tile_store.glslh (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => pack_weights}/pack_int4_linear_weight_transposed_interleaved.glsl (100%) rename backends/vulkan/runtime/graph/ops/glsl/{ => pack_weights}/pack_int4_linear_weight_transposed_interleaved.yaml (100%) delete mode 100644 backends/vulkan/runtime/graph/ops/impl/GemmCommon.h delete mode 100644 backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.h rename backends/vulkan/runtime/graph/ops/impl/{ => gemm}/GemmCommon.cpp (98%) rename backends/vulkan/runtime/graph/ops/impl/{GemmCoopmat.h => gemm/GemmCommon.h} (59%) rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/linear}/Linear.cpp (95%) rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/linear}/Linear.h (100%) rename backends/vulkan/runtime/graph/ops/impl/{GemmCoopmat.cpp => gemm/linear/LinearCoopmat.cpp} (53%) create mode 100644 backends/vulkan/runtime/graph/ops/impl/gemm/linear/LinearCoopmat.h rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/linear/backward}/LinearDw.cpp (100%) rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/linear/q8ta}/Q8taLinear.cpp (97%) rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/linear/q8ta}/Q8taLinear.h (100%) rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/linear/q8ta}/Q8taLinearGemv.cpp (97%) rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/linear/q8ta}/Q8taLinearGemv.h (100%) rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/linear/qw}/QuantizedLinear.cpp (63%) create mode 100644 backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/QuantizedLinear.h create mode 100644 backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.cpp create mode 100644 backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.h create mode 100644 backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.cpp create mode 100644 backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.h rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/linear/qw/legacy}/QuantizedLinearQCSNW.cpp (99%) rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/linear/qw/q4gsw}/Q4gswLinear.cpp (99%) rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/linear/qw/q4gsw}/Q4gswLinear.h (100%) rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/linear/qw/q4gsw/backward}/QuantizedLinearBackward.cpp (97%) rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/linear/qw/q4gsw/backward}/QuantizedLinearRequant.cpp (100%) create mode 100644 backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q8csw/Q8cswLinear.cpp rename backends/vulkan/runtime/graph/ops/impl/{Matmul.cpp => gemm/matmul/MatMul.cpp} (95%) rename backends/vulkan/runtime/graph/ops/impl/{ => gemm/matmul}/MatMul.h (100%) create mode 100644 backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.cpp create mode 100644 backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.h diff --git a/backends/vulkan/op_registry.py b/backends/vulkan/op_registry.py index 22c9da4aab1..019c141b6fc 100644 --- a/backends/vulkan/op_registry.py +++ b/backends/vulkan/op_registry.py @@ -417,7 +417,7 @@ def register_softmax_cpp_ops(): # ============================================================================= -# MatMul.cpp +# gemm/matmul/MatMul.cpp # ============================================================================= @@ -437,7 +437,7 @@ def register_matmul_cpp_ops(): # ============================================================================= -# Linear.cpp +# gemm/linear/Linear.cpp # ============================================================================= @@ -457,7 +457,7 @@ def register_linear_cpp_ops(): # ============================================================================= -# QuantizedLinearQCSNW.cpp +# gemm/linear/qw/{q8csw,legacy}/*.cpp # ============================================================================= @@ -477,7 +477,7 @@ def register_quantizedlinearqcsnw_cpp_ops(): # ============================================================================= -# QuantizedLinear.cpp +# gemm/linear/qw/{q4gsw,dq8ca_q4gsw,legacy}/**/*.cpp # ============================================================================= @@ -1092,7 +1092,7 @@ def register_q8ta_conv2d_transposed_op(): # ============================================================================= -# Q8taLinear.cpp +# gemm/linear/q8ta/Q8taLinear.cpp # ============================================================================= diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_input_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_input_tile_load.glslh index 4456043bb9f..f1cafc503e3 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_input_tile_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_input_tile_load.glslh @@ -11,7 +11,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" VEC4_T load_fp_input_texel(const Conv2dTensorIndex tidx) { #ifdef INPUT_BUFFER diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_pw.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_pw.glsl index 4ebca126e03..41d422515f2 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_pw.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_pw.glsl @@ -67,11 +67,11 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "linear_fp_input_tile.glslh" -#include "linear_fp_weight_tile.glslh" -#include "linear_fp_output_tile.glslh" -#include "linear_fp_packed_weight_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_weight_tile.glslh" +#include "gemm/tile_utils/linear_fp_output_tile.glslh" +#include "gemm/tile_utils/linear_fp_packed_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" // Conv1d pointwise is matrix multiplication with swapped texture coordinates. // Linear: input ivec3(k4, m, b), output ivec3(n4, m, b) [width-packed] diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_tiled.glsl index bd12d149a2c..e4e8ae7a5ff 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_tiled.glsl @@ -47,9 +47,9 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; ${layout_declare_spec_const(C, "int", "stride_1_padding_0", "0")} ${layout_declare_spec_const(C, "int", "activation_type", "0")} -#include "linear_fp_input_tile.glslh" -#include "linear_fp_packed_weight_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_packed_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" // Texel coordinates for the TILE_M spatial positions this invocation covers. // diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_gemm.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_gemm.glsl index 6eebdb6b25f..d704b2363bf 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_gemm.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_gemm.glsl @@ -114,9 +114,9 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; ${layout_declare_spec_const(C, "int", "activation_type", "0")} -#include "linear_fp_input_tile.glslh" -#include "linear_fp_packed_weight_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_packed_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" /* * Load TILE_M rows × TILE_K4 K-tiles of the im2col'd input. diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_load.glslh index b94bdaf506f..d156adc6e6a 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_load.glslh @@ -26,7 +26,7 @@ #include "indexing.glslh" #include "conv2d_common.glslh" #include "conv2d_fp_im2col_block.glslh" -#include "linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" VEC4_T load_input_texel(const TensorIndex4D tidx) { // Assumes batch size is 1 and channels packing diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_store.glslh index 6c4dd7f0b52..8859b6c02e3 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_store.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_store.glslh @@ -23,7 +23,7 @@ #include "indexing.glslh" #include "conv2d_common.glslh" #include "conv2d_fp_im2col_block.glslh" -#include "linear_fp_output_tile.glslh" +#include "gemm/tile_utils/linear_fp_output_tile.glslh" // TODO: implement buffer support void write_output_texel(const VEC4_T out_texel, const TensorIndex4D tidx) { diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_input_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_input_tile_load.glslh index 44aa09912ec..69407633e1f 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_input_tile_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_input_tile_load.glslh @@ -11,7 +11,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_int8_input_tile.glslh" +#include "gemm/tile_utils/linear_int8_input_tile.glslh" struct Int8InputTileIndex { #ifdef PACKED_INT8_INPUT_BUFFER diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_output_tile_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_output_tile_store.glslh index 27244f67953..e6c48d99462 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_output_tile_store.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_output_tile_store.glslh @@ -12,7 +12,7 @@ #extension GL_EXT_control_flow_attributes : require #include "conv2d_common.glslh" -#include "linear_int8_output_tile.glslh" +#include "gemm/tile_utils/linear_int8_output_tile.glslh" void store_packed_int8_output_tile( const Int8OutTile int8_tile, diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl index f52241ee60f..bff8267c412 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl @@ -74,12 +74,12 @@ ${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")} #include "conv2d_int8_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_bias_load.glslh" -#include "linear_int8_output_tile_compute.glslh" +#include "gemm/tile_utils/linear_int8_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "gemm/tile_utils/linear_int_weight_sums_load.glslh" +#include "gemm/tile_utils/linear_fp_weight_scales_load.glslh" +#include "gemm/tile_utils/linear_fp_bias_load.glslh" +#include "gemm/tile_utils/linear_int8_output_tile_compute.glslh" #include "conv2d_int8_output_tile_store.glslh" void main() { diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8_utils.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8_utils.glslh index 3fa03978fed..c30cf23aad8 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8_utils.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8_utils.glslh @@ -12,7 +12,7 @@ #extension GL_EXT_control_flow_attributes : require #extension GL_EXT_integer_dot_product : require -#include "linear_int_accumulator.glslh" +#include "gemm/tile_utils/linear_int_accumulator.glslh" struct Int8InputWindow1D { int[MAX_WINDOW_WIDTH] data; diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8csw_linear_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8csw_linear_tiled.glsl index a93b34a3e6e..f0dcaa88273 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8csw_linear_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8csw_linear_tiled.glsl @@ -63,12 +63,12 @@ ${layout_declare_spec_const(C, "int", "conv2d_params_logical_K_per_group", "1")} ${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")} -#include "linear_fp_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_bias_load.glslh" -#include "linear_fp_output_tile_fp_int8_compute.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_input_tile_load.glslh" +#include "gemm/tile_utils/linear_int8_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_weight_scales_load.glslh" +#include "gemm/tile_utils/linear_fp_bias_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_int8_compute.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" #include "conv2d_fp_im2col_block_store.glslh" void main() { diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_linear_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_linear_tiled.glsl index 3500f8392ad..dbbf3a08d2c 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_linear_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_linear_tiled.glsl @@ -71,12 +71,12 @@ ${layout_declare_spec_const(C, "int", "conv2d_params_logical_K_per_group", "1")} ${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")} -#include "linear_int8_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_fp_bias_load.glslh" +#include "gemm/tile_utils/linear_int8_input_tile_load.glslh" +#include "gemm/tile_utils/linear_int8_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_int_weight_sums_load.glslh" +#include "gemm/tile_utils/linear_fp_weight_scales_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "gemm/tile_utils/linear_fp_bias_load.glslh" #include "conv2d_fp_im2col_block_store.glslh" void main() { diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to.glsl index 2c7f0aab981..4e3da29355b 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to.glsl @@ -77,12 +77,12 @@ ${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")} #include "im2col_packed_int8_utils.glslh" #include "conv2d_int8_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_bias_load.glslh" -#include "linear_int8_output_tile_compute.glslh" +#include "gemm/tile_utils/linear_int8_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "gemm/tile_utils/linear_int_weight_sums_load.glslh" +#include "gemm/tile_utils/linear_fp_weight_scales_load.glslh" +#include "gemm/tile_utils/linear_fp_bias_load.glslh" +#include "gemm/tile_utils/linear_int8_output_tile_compute.glslh" #include "conv2d_int8_output_tile_store.glslh" #include "conv2d_q8_utils.glslh" diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl index f136c6ce99a..b46a0630862 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl @@ -74,12 +74,12 @@ ${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")} #include "conv2d_int8_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_bias_load.glslh" -#include "linear_int8_output_tile_compute.glslh" +#include "gemm/tile_utils/linear_int8_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "gemm/tile_utils/linear_int_weight_sums_load.glslh" +#include "gemm/tile_utils/linear_fp_weight_scales_load.glslh" +#include "gemm/tile_utils/linear_fp_bias_load.glslh" +#include "gemm/tile_utils/linear_int8_output_tile_compute.glslh" #include "conv2d_int8_output_tile_store.glslh" void main() { diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/quantize_and_pack_im2col.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/quantize_and_pack_im2col.glsl index a008866a779..1011b6d41ac 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/quantize_and_pack_im2col.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/quantize_and_pack_im2col.glsl @@ -67,7 +67,7 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "conv2d_fp_im2col_block_load.glslh" -#include "linear_int8_input_block.glslh" +#include "gemm/tile_utils/linear_int8_input_block.glslh" void main() { // The quantized and packed im2col matrix can be conceptualized as a 2D matrix diff --git a/backends/vulkan/runtime/graph/ops/glsl/coopmat_mm.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/coopmat_mm.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/coopmat_mm.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/coopmat_mm.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/coopmat_mm.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/coopmat_mm.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/coopmat_mm.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/coopmat_mm.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_dW.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/backward/linear_dW.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_dW.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/backward/linear_dW.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_dW.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/backward/linear_dW.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_dW.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/backward/linear_dW.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_scalar.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_scalar.glsl similarity index 90% rename from backends/vulkan/runtime/graph/ops/glsl/linear_scalar.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_scalar.glsl index 9f5c2ee6a50..4ac5b9c88ee 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_scalar.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_scalar.glsl @@ -61,11 +61,11 @@ $else: layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "matmul_fp_mat1_tile_load.glslh" -#include "linear_fp_packed_weight_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "matmul_fp_out_tile_store.glslh" -#include "matmul_fp_bias_apply.glslh" +#include "tile_utils/matmul_fp_mat1_tile_load.glslh" +#include "tile_utils/linear_fp_packed_weight_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/matmul_fp_out_tile_store.glslh" +#include "tile_utils/matmul_fp_bias_apply.glslh" void main() { const int tile_idx_n = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_scalar.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_scalar.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_scalar.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_scalar.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_vec.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_vec.glsl similarity index 90% rename from backends/vulkan/runtime/graph/ops/glsl/linear_vec.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_vec.glsl index adf5272673e..f17520777f6 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_vec.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_vec.glsl @@ -62,11 +62,11 @@ $else: layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "matmul_fp_mat1_tile_load.glslh" -#include "linear_fp_packed_weight_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "matmul_fp_out_tile_store.glslh" -#include "matmul_fp_bias_apply.glslh" +#include "tile_utils/matmul_fp_mat1_tile_load.glslh" +#include "tile_utils/linear_fp_packed_weight_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/matmul_fp_out_tile_store.glslh" +#include "tile_utils/matmul_fp_bias_apply.glslh" void main() { const int tile_idx_n = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_vec.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_vec.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_vec.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_vec.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_fp_linear_weight.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/pack_weights/pack_fp_linear_weight.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_fp_linear_weight.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/pack_weights/pack_fp_linear_weight.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_fp_linear_weight.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/pack_weights/pack_fp_linear_weight.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_fp_linear_weight.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/pack_weights/pack_fp_linear_weight.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_linear.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear.glsl similarity index 92% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_linear.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear.glsl index 98b0895d759..b09b593990c 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/q8ta_linear.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear.glsl @@ -57,12 +57,12 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "common.glslh" -#include "linear_int8_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_bias_load.glslh" +#include "tile_utils/linear_int8_input_tile_load.glslh" +#include "tile_utils/linear_int8_weight_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_int_weight_sums_load.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" void main() { const int out_tile_x = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_linear.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_linear.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_linear_gemv.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear_gemv.glsl similarity index 94% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_linear_gemv.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear_gemv.glsl index 9153cb0e294..7ae85518563 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/q8ta_linear_gemv.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear_gemv.glsl @@ -62,11 +62,11 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "dispatch.glslh" #include "common.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_bias_load.glslh" +#include "tile_utils/linear_int8_weight_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_int_weight_sums_load.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" // Array-of-arrays shared memory layout: partial_accums[lid][tile_n4]. // Each element is exactly 16 bytes (ivec4). This avoids the Samsung S25 diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_linear_gemv.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear_gemv.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_linear_gemv.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear_gemv.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_q4gsw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.glsl similarity index 85% rename from backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_q4gsw_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.glsl index a68a56c3713..5acab8e9a12 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_q4gsw_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.glsl @@ -60,19 +60,19 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; ${layout_declare_spec_const(C, "int", "apply_bias", "0")} ${layout_declare_spec_const(C, "int", "K4_per_group", "0")} -#include "linear_fp_input_tile_load.glslh" -#include "linear_int8_input_tile_load.glslh" -#include "linear_int8_input_scales_zps_load.glslh" -#include "linear_int4_weight_tile_load.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_int8_input_sums_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_fp_output_tile_int8_int4_compute.glslh" -#include "linear_fp_output_tile_fp_int4_compute.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "linear_fp_output_tile_store.glslh" -#include "linear_fp_bias_load.glslh" +#include "tile_utils/linear_fp_input_tile_load.glslh" +#include "tile_utils/linear_int8_input_tile_load.glslh" +#include "tile_utils/linear_int8_input_scales_zps_load.glslh" +#include "tile_utils/linear_int4_weight_tile_load.glslh" +#include "tile_utils/linear_int_weight_sums_load.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_int8_input_sums_load.glslh" +#include "tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "tile_utils/linear_fp_output_tile_int8_int4_compute.glslh" +#include "tile_utils/linear_fp_output_tile_fp_int4_compute.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/linear_fp_output_tile_store.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" void main() { const int out_tile_x = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_q4gsw_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_q4gsw_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_qw_coopmat.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_qw_coopmat.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_qw_coopmat.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_qw_coopmat.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_qw_coopmat.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_qw_coopmat.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_qw_coopmat.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_qw_coopmat.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q8ta_q8csw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_q8ta_q8csw_tiled.glsl similarity index 89% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q8ta_q8csw_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_q8ta_q8csw_tiled.glsl index e3f49fac475..20003489dca 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_q8ta_q8csw_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_q8ta_q8csw_tiled.glsl @@ -50,13 +50,13 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "linear_int8_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_fp_output_tile_store.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_bias_load.glslh" +#include "tile_utils/linear_int8_input_tile_load.glslh" +#include "tile_utils/linear_int8_weight_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "tile_utils/linear_fp_output_tile_store.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_int_weight_sums_load.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" void main() { // Each thread writes out a 4 wide x 4 high tile of output values diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q8ta_q8csw_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_q8ta_q8csw_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q8ta_q8csw_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_q8ta_q8csw_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_coop.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_coop.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_coop.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_coop.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_coop.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_coop.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_coop.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_coop.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_tiled.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_tiled.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_linear_weight.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/pack_weights/pack_q8_linear_weight.glsl similarity index 97% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q8_linear_weight.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/pack_weights/pack_q8_linear_weight.glsl index 7b17dbe4990..765c8a164e1 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_linear_weight.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/pack_weights/pack_q8_linear_weight.glsl @@ -26,7 +26,7 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "common.glslh" -#include "linear_int8_weight_block.glslh" +#include "tile_utils/linear_int8_weight_block.glslh" void main() { // The size of the source weight tensor is [W=K, H=N]. Each shader invocation diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_linear_weight.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/pack_weights/pack_q8_linear_weight.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q8_linear_weight.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/pack_weights/pack_q8_linear_weight.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_backward.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_backward.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_backward.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_backward.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_backward.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_backward.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_backward.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_backward.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_requant.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_requant.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_requant.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_requant.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_requant.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_requant.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_requant.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_requant.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_coop.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_coop.glsl similarity index 92% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_coop.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_coop.glsl index 0f88dc59702..8a2c9f8bdcf 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_coop.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_coop.glsl @@ -63,13 +63,13 @@ ${layout_declare_spec_const(C, "int", "apply_bias", "0")} ${layout_declare_spec_const(C, "int", "K4_per_group", "0")} #include "common.glslh" -#include "linear_fp_input_tile_load.glslh" -#include "linear_int4_weight_tile_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_output_tile_fp_int4_compute.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "linear_fp_output_tile_store.glslh" -#include "linear_fp_bias_load.glslh" +#include "tile_utils/linear_fp_input_tile_load.glslh" +#include "tile_utils/linear_int4_weight_tile_load.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_fp_output_tile_fp_int4_compute.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/linear_fp_output_tile_store.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" shared FPOutTile partial_sums[WGS]; diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_coop.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_coop.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_coop.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_coop.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_tiled.glsl similarity index 89% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_tiled.glsl index f79512ac7ee..9bfa875d58d 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_tiled.glsl @@ -49,13 +49,13 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; ${layout_declare_spec_const(C, "int", "apply_bias", "0")} ${layout_declare_spec_const(C, "int", "K4_per_group", "0")} -#include "linear_fp_input_tile_load.glslh" -#include "linear_int4_weight_tile_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_bias_load.glslh" -#include "linear_fp_output_tile_fp_int4_compute.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "linear_fp_output_tile_store.glslh" +#include "tile_utils/linear_fp_input_tile_load.glslh" +#include "tile_utils/linear_int4_weight_tile_load.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" +#include "tile_utils/linear_fp_output_tile_fp_int4_compute.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/linear_fp_output_tile_store.glslh" void main() { const int out_tile_x = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qw_coopmat.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_qw_coopmat.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qw_coopmat.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_qw_coopmat.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qw_coopmat.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_qw_coopmat.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qw_coopmat.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_qw_coopmat.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight.glsl similarity index 97% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight.glsl index b9f5c994910..2b2cf023b5c 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight.glsl @@ -25,7 +25,7 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "common.glslh" -#include "linear_int4_weight_block.glslh" +#include "tile_utils/linear_int4_weight_block.glslh" void main() { const int k8 = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight__w_4x8.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight__w_4x8.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight__w_4x8.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight__w_4x8.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight__w_4x8.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight__w_4x8.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight__w_4x8.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight__w_4x8.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__tin__w_4x8.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__tin__w_4x8.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__tin__w_4x8.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__tin__w_4x8.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__tin__w_4x8.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__tin__w_4x8.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__tin__w_4x8.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__tin__w_4x8.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__w_4x8.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__w_4x8.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__w_4x8.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__w_4x8.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__w_4x8.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__w_4x8.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__w_4x8.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__w_4x8.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemv_coop__w_4x8.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemv_coop__w_4x8.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemv_coop__w_4x8.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemv_coop__w_4x8.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemv_coop__w_4x8.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemv_coop__w_4x8.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemv_coop__w_4x8.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemv_coop__w_4x8.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q8csw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q8csw/linear_q8csw_tiled.glsl similarity index 87% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q8csw_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q8csw/linear_q8csw_tiled.glsl index c8dc6bfeec4..51a73d1126f 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_q8csw_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q8csw/linear_q8csw_tiled.glsl @@ -44,14 +44,14 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; ${layout_declare_spec_const(C, "uint", "apply_bias", "0")} -#include "linear_fp_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_weight_tile.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "linear_fp_output_tile_fp_int8_compute.glslh" -#include "linear_fp_output_tile_store.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_bias_load.glslh" +#include "tile_utils/linear_fp_input_tile_load.glslh" +#include "tile_utils/linear_int8_weight_tile_load.glslh" +#include "tile_utils/linear_fp_weight_tile.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/linear_fp_output_tile_fp_int8_compute.glslh" +#include "tile_utils/linear_fp_output_tile_store.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" void main() { // Each thread writes out a 4 wide x 4 high tile of output values diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q8csw_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q8csw/linear_q8csw_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q8csw_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q8csw/linear_q8csw_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_scalar.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_scalar.glsl similarity index 90% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_scalar.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_scalar.glsl index a3f68dda839..bfffe5fa38d 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/matmul_scalar.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_scalar.glsl @@ -54,11 +54,11 @@ $if HAS_BIAS: layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "matmul_fp_mat1_tile_load.glslh" -#include "matmul_fp_mat2_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "matmul_fp_out_tile_store.glslh" -#include "matmul_fp_bias_apply.glslh" +#include "tile_utils/matmul_fp_mat1_tile_load.glslh" +#include "tile_utils/matmul_fp_mat2_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/matmul_fp_out_tile_store.glslh" +#include "tile_utils/matmul_fp_bias_apply.glslh" void main() { const int tile_idx_n = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_scalar.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_scalar.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_scalar.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_scalar.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_vec.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_vec.glsl similarity index 90% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_vec.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_vec.glsl index 9d4f6f04aba..c14861170c2 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/matmul_vec.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_vec.glsl @@ -55,11 +55,11 @@ $if HAS_BIAS: layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "matmul_fp_mat1_tile_load.glslh" -#include "matmul_fp_mat2_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "matmul_fp_out_tile_store.glslh" -#include "matmul_fp_bias_apply.glslh" +#include "tile_utils/matmul_fp_mat1_tile_load.glslh" +#include "tile_utils/matmul_fp_mat2_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/matmul_fp_out_tile_store.glslh" +#include "tile_utils/matmul_fp_bias_apply.glslh" void main() { const int tile_idx_n = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_vec.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_vec.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_vec.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_vec.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_common.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_common.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_common.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_common.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_bias_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_bias_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_bias_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_bias_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_input_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_input_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_input_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_input_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_input_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_input_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_input_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_input_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_fp_compute.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_fp_compute.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_fp_int4_compute.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_fp_int4_compute.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_fp_int4_compute.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_fp_int4_compute.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_fp_int8_compute.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_fp_int8_compute.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_fp_int8_compute.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_fp_int8_compute.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_int8_int4_compute.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_int8_int4_compute.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_int8_int4_compute.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_int8_int4_compute.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_int8_int8_compute.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_int8_int8_compute.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_int8_int8_compute.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_int8_int8_compute.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_store.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_store.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_store.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_packed_weight_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_packed_weight_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_packed_weight_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_packed_weight_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_per_out_channel_params.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_per_out_channel_params.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_per_out_channel_params.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_per_out_channel_params.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_weight_scales_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_weight_scales_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_weight_scales_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_weight_scales_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_weight_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_weight_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_weight_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_weight_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int4_weight_block.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int4_weight_block.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int4_weight_block.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int4_weight_block.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int4_weight_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int4_weight_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int4_weight_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int4_weight_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int4_weight_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int4_weight_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int4_weight_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int4_weight_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_block.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_block.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_block.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_block.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_scales_zps.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_scales_zps.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_scales_zps.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_scales_zps.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_scales_zps_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_scales_zps_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_scales_zps_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_scales_zps_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_sums_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_sums_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_sums_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_sums_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_sums_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_sums_store.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_sums_store.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_sums_store.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_output_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_output_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_output_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_output_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_output_tile_compute.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_output_tile_compute.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_output_tile_compute.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_output_tile_compute.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_weight_block.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_weight_block.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_weight_block.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_weight_block.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_weight_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_weight_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_weight_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_weight_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_weight_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_weight_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_weight_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_weight_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int_accumulator.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_accumulator.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int_accumulator.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_accumulator.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int_per_in_channel_params.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_per_in_channel_params.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int_per_in_channel_params.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_per_in_channel_params.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int_per_out_channel_params.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_per_out_channel_params.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int_per_out_channel_params.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_per_out_channel_params.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int_weight_sums_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_weight_sums_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int_weight_sums_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_weight_sums_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_fp_bias_apply.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_bias_apply.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_fp_bias_apply.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_bias_apply.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_fp_mat1_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_mat1_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_fp_mat1_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_mat1_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_fp_mat2_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_mat2_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_fp_mat2_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_mat2_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_fp_out_tile_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_out_tile_store.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_fp_out_tile_store.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_out_tile_store.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_int4_linear_weight_transposed_interleaved.glsl b/backends/vulkan/runtime/graph/ops/glsl/pack_weights/pack_int4_linear_weight_transposed_interleaved.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_int4_linear_weight_transposed_interleaved.glsl rename to backends/vulkan/runtime/graph/ops/glsl/pack_weights/pack_int4_linear_weight_transposed_interleaved.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_int4_linear_weight_transposed_interleaved.yaml b/backends/vulkan/runtime/graph/ops/glsl/pack_weights/pack_int4_linear_weight_transposed_interleaved.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_int4_linear_weight_transposed_interleaved.yaml rename to backends/vulkan/runtime/graph/ops/glsl/pack_weights/pack_int4_linear_weight_transposed_interleaved.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w.glsl b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w.glsl index 9d65d7d534f..41414a22dfd 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w.glsl @@ -38,8 +38,8 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "linear_int8_input_block.glslh" -#include "linear_fp_input_tile_load.glslh" +#include "gemm/tile_utils/linear_int8_input_block.glslh" +#include "gemm/tile_utils/linear_fp_input_tile_load.glslh" void main() { // Each input block contains 4x4 int8 quantized values, which are packed into diff --git a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w_with_group_sums.glsl b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w_with_group_sums.glsl index d5f14f70b62..407c49fddf4 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w_with_group_sums.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w_with_group_sums.glsl @@ -48,9 +48,9 @@ shared ivec4 shared_sums[NUM_GROUPS_PER_WG][NUM_WORKERS_PER_GROUP]; #define TILE_M 4 -#include "linear_int8_input_block.glslh" -#include "linear_int8_input_scales_zps_load.glslh" -#include "linear_fp_input_tile_load.glslh" +#include "gemm/tile_utils/linear_int8_input_block.glslh" +#include "gemm/tile_utils/linear_int8_input_scales_zps_load.glslh" +#include "gemm/tile_utils/linear_fp_input_tile_load.glslh" void main() { const int group_idx = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4w4c.glsl b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4w4c.glsl index 6f58021be7f..3b5be301425 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4w4c.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4w4c.glsl @@ -43,7 +43,7 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "conv2d_fp_input_tile_load.glslh" -#include "linear_int8_input_block.glslh" +#include "gemm/tile_utils/linear_int8_input_block.glslh" void store_packed_int8_block( const Conv2dBlockIndex block_idx, diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_coop.glsl b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_coop.glsl index b7f14f435fa..ea1a80d6388 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_coop.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_coop.glsl @@ -51,7 +51,7 @@ ${layout_declare_spec_const(C, "float", "inv_scale", "1.0")} #include "sdpa_fp_q_projected_tile_load.glslh" #include "sdpa_fp_k_cache_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" #include "sdpa_fp_attn_weight_tile_store.glslh" shared FPOutTile partial_sums[NUM_WORKERS_PER_OUT]; diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_tiled.glsl index 662d5edde68..a0feba5bb15 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_tiled.glsl @@ -70,7 +70,7 @@ ${layout_declare_spec_const(C, "float", "inv_scale", "1.0")} #include "sdpa_fp_q_projected_tile_load.glslh" #include "sdpa_fp_k_cache_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" #include "sdpa_fp_attn_weight_tile_store.glslh" /* diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_coop.glsl b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_coop.glsl index 7f873ce0718..d1aab16290f 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_coop.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_coop.glsl @@ -64,7 +64,7 @@ $if GQA: #include "sdpa_fp_attn_weight_tile_load.glslh" #include "sdpa_fp_v_cache_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" #include "sdpa_fp_out_tile_store.glslh" #ifdef GQA diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_tiled.glsl index 9f8f2dbc231..d191401296c 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_tiled.glsl @@ -56,7 +56,7 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "sdpa_fp_attn_weight_tile_load.glslh" #include "sdpa_fp_v_cache_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" #include "sdpa_fp_out_tile_store.glslh" /* diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_load.glslh index 829f03beb60..af639e77371 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_load.glslh @@ -27,7 +27,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" LINEAR_FP_INPUT_TILE_VEC4_T load_attn_weight_c4( const int c4, diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_store.glslh index 83d81ddf812..78a7df1d4be 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_store.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_store.glslh @@ -32,7 +32,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_output_tile.glslh" +#include "gemm/tile_utils/linear_fp_output_tile.glslh" // ============================================================ // Shared store helpers (buffer or texture; fp32 or input-dtype) diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_k_cache_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_k_cache_tile_load.glslh index 65d08755528..555561581c7 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_k_cache_tile_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_k_cache_tile_load.glslh @@ -31,7 +31,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_weight_tile.glslh" +#include "gemm/tile_utils/linear_fp_weight_tile.glslh" // Determine whether buffer mode is active. Both K_CACHE_BUFFER (LLM) and // K_BUFFER (fused) activate the buffer path. diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_out_tile_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_out_tile_store.glslh index 382747bf9a4..2c04170bab4 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_out_tile_store.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_out_tile_store.glslh @@ -30,7 +30,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_output_tile.glslh" +#include "gemm/tile_utils/linear_fp_output_tile.glslh" void store_out_d4( const LINEAR_FP_OUTPUT_TILE_VEC4_T out_texel, diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_q_projected_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_q_projected_tile_load.glslh index 752762b623d..40ed5b9e181 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_q_projected_tile_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_q_projected_tile_load.glslh @@ -28,7 +28,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" LINEAR_FP_INPUT_TILE_VEC4_T load_q_projected_d4( const int d4, diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_v_cache_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_v_cache_tile_load.glslh index 98516744b44..fc2b280dedd 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_v_cache_tile_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_v_cache_tile_load.glslh @@ -30,7 +30,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_weight_tile.glslh" +#include "gemm/tile_utils/linear_fp_weight_tile.glslh" #ifndef SDPA_V_BUF #define SDPA_V_BUF t_v diff --git a/backends/vulkan/runtime/graph/ops/glsl/unpack_4w4c_and_dequantize.glsl b/backends/vulkan/runtime/graph/ops/glsl/unpack_4w4c_and_dequantize.glsl index fd7fa0aa7f1..41a106ab5ff 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/unpack_4w4c_and_dequantize.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/unpack_4w4c_and_dequantize.glsl @@ -42,8 +42,8 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "linear_fp_input_tile.glslh" -#include "linear_int8_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_int8_input_tile.glslh" void load_packed_int8_tile( out Int8InputTile int8_tile, diff --git a/backends/vulkan/runtime/graph/ops/impl/EmbeddingQ4gsw.cpp b/backends/vulkan/runtime/graph/ops/impl/EmbeddingQ4gsw.cpp index ad4786d1c60..2d0d8c623f4 100644 --- a/backends/vulkan/runtime/graph/ops/impl/EmbeddingQ4gsw.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/EmbeddingQ4gsw.cpp @@ -9,8 +9,8 @@ #include #include -#include #include +#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/GemmCommon.h b/backends/vulkan/runtime/graph/ops/impl/GemmCommon.h deleted file mode 100644 index eadeeee7080..00000000000 --- a/backends/vulkan/runtime/graph/ops/impl/GemmCommon.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include -#include - -namespace vkcompute { - -// Helpers shared by the linear/matmul tiled and coopmat dispatch paths. - -// Prepack a floating-point weight tensor for the linear/matmul kernels. -// Source layout: [N, K] (is_transposed=true) or [K, N] (is_transposed=false), -// optionally batched. Output layout: 4OC x 4IC blocked, packed as -// kWidthPacked. When force_buffer is true, the packed tensor uses buffer -// storage (required by the coopmat shader); otherwise texture2d is used -// when it fits within max_texture2d_dim. -ValueRef prepack_fp_linear_weight( - ComputeGraph& graph, - const ValueRef weight_data, - bool is_transposed, - int64_t B, - bool force_buffer = false); - -// Resize logic for linear-shaped output: takes M from the input's penultimate -// dim, N from resize_args[0], optional batch from input's leading dim. -void resize_linear_node( - ComputeGraph* graph, - const std::vector& args, - const std::vector& resize_args); - -// Resize logic for matmul (mat1 @ mat2): output rows from mat1, cols from -// mat2, batch dims propagated from mat1. -void resize_matmul_tiled_node( - ComputeGraph* graph, - const std::vector& args, - const std::vector& resize_args); - -} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.h b/backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.h deleted file mode 100644 index 4ab87f8ac33..00000000000 --- a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include -#include -#include - -namespace vkcompute { - -LocalWorkGroup quantized_linear_lwg( - ComputeGraph* graph, - const vkapi::ShaderInfo& shader, - const GlobalWorkGrid& gwg, - const std::vector& args, - const std::vector& resize_args); - -ValueRef prepack_quantized_linear_weight( - ComputeGraph& graph, - const QuantizationConfig& weight_quant_config, - const ValueRef qmat2_data, - const bool use_unsigned_dot = false); - -} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/SDPA.cpp b/backends/vulkan/runtime/graph/ops/impl/SDPA.cpp index 39854b23d02..bc315727bd2 100644 --- a/backends/vulkan/runtime/graph/ops/impl/SDPA.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/SDPA.cpp @@ -12,10 +12,9 @@ #include #include -#include #include #include -#include +#include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dPW.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dPW.cpp index 225a5118688..22457c37e24 100644 --- a/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dPW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dPW.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.cpp index 39f9a44b476..0fcb2105b2e 100644 --- a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.cpp @@ -14,9 +14,9 @@ #include #include -#include #include #include +#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.cpp index 595395b6f82..d58f669362f 100644 --- a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.cpp @@ -11,13 +11,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include #include namespace vkcompute { diff --git a/backends/vulkan/runtime/graph/ops/impl/GemmCommon.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/GemmCommon.cpp similarity index 98% rename from backends/vulkan/runtime/graph/ops/impl/GemmCommon.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/GemmCommon.cpp index b763dcaeb38..61e053e0751 100644 --- a/backends/vulkan/runtime/graph/ops/impl/GemmCommon.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/GemmCommon.cpp @@ -6,7 +6,7 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/GemmCoopmat.h b/backends/vulkan/runtime/graph/ops/impl/gemm/GemmCommon.h similarity index 59% rename from backends/vulkan/runtime/graph/ops/impl/GemmCoopmat.h rename to backends/vulkan/runtime/graph/ops/impl/gemm/GemmCommon.h index 7be7e8bc157..eb1c1c74c24 100644 --- a/backends/vulkan/runtime/graph/ops/impl/GemmCoopmat.h +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/GemmCommon.h @@ -9,13 +9,40 @@ #pragma once #include +#include namespace vkcompute { -// Cooperative-matrix GEMM dispatch. The coopmat shader (coopmat_mm.glsl) is -// shared across linear/matmul; the two add_*_node entry points below differ -// only in input layout (prepacked weight vs runtime mat2) and bias handling. -// +// Helpers shared by the linear/matmul tiled and coopmat dispatch paths. + +// Prepack a floating-point weight tensor for the linear/matmul kernels. +// Source layout: [N, K] (is_transposed=true) or [K, N] (is_transposed=false), +// optionally batched. Output layout: 4OC x 4IC blocked, packed as +// kWidthPacked. When force_buffer is true, the packed tensor uses buffer +// storage (required by the coopmat shader); otherwise texture2d is used +// when it fits within max_texture2d_dim. +ValueRef prepack_fp_linear_weight( + ComputeGraph& graph, + const ValueRef weight_data, + bool is_transposed, + int64_t B, + bool force_buffer = false); + +// Resize logic for linear-shaped output: takes M from the input's penultimate +// dim, N from resize_args[0], optional batch from input's leading dim. +void resize_linear_node( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args); + +// Resize logic for matmul (mat1 @ mat2): output rows from mat1, cols from +// mat2, batch dims propagated from mat1. +void resize_matmul_tiled_node( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args); + +// Cooperative-matrix GEMM (coopmat_mm.glsl, shared by linear and matmul). // Tile dimensions match the coopmat_mm.yaml defaults; they participate in // the eligibility check below because the shader has no partial-tile or // K-tail handling — misaligned shapes must fall back to the tiled path. @@ -52,19 +79,4 @@ inline bool is_coopmat_eligible( N % kCoopmatTileN == 0 && K % kCoopmatTileK == 0; } -void add_linear_coopmat_node( - ComputeGraph& graph, - const ValueRef input, - const ValueRef packed_weight, - const ValueRef packed_bias, - bool has_bias, - const ValueRef out, - int32_t weight_B = 1); - -void add_matmul_coopmat_node( - ComputeGraph& graph, - const ValueRef mat1, - const ValueRef mat2, - const ValueRef out); - } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Linear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/Linear.cpp similarity index 95% rename from backends/vulkan/runtime/graph/ops/impl/Linear.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/Linear.cpp index 86f27b63586..722cde98515 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Linear.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/Linear.cpp @@ -9,11 +9,11 @@ #include #include -#include -#include -#include -#include #include +#include +#include +#include +#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/Linear.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/Linear.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Linear.h rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/Linear.h diff --git a/backends/vulkan/runtime/graph/ops/impl/GemmCoopmat.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/LinearCoopmat.cpp similarity index 53% rename from backends/vulkan/runtime/graph/ops/impl/GemmCoopmat.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/LinearCoopmat.cpp index 6b62fa6a9ba..5b3e898cff6 100644 --- a/backends/vulkan/runtime/graph/ops/impl/GemmCoopmat.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/LinearCoopmat.cpp @@ -6,19 +6,15 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include -#include +#include -#include -#include #include namespace vkcompute { -// ── Linear coopmat ── - static vkapi::ShaderInfo pick_linear_coopmat_shader( ComputeGraph* graph, const std::vector& args, @@ -40,8 +36,10 @@ static GlobalWorkGrid pick_linear_coopmat_gwg( (void)resize_args; const ValueRef out = args.at(0).refs.at(0); const auto out_sizes = graph->sizes_of(out); - uint32_t M = out_sizes.at(out_sizes.size() - 2); - uint32_t N = out_sizes.at(out_sizes.size() - 1); + uint32_t M = + utils::safe_downcast(out_sizes.at(out_sizes.size() - 2)); + uint32_t N = + utils::safe_downcast(out_sizes.at(out_sizes.size() - 1)); uint32_t num_tiles_n = utils::div_up(N, kCoopmatTileN); uint32_t num_tiles_m = utils::div_up(M, kCoopmatTileM); // Each workgroup processes one WG_TILE_M x WG_TILE_N output tile via @@ -81,7 +79,8 @@ void add_linear_coopmat_node( "linear_coopmat requires buffer storage"); std::vector out_sizes = graph.sizes_of(out); - int32_t orig_N = utils::safe_downcast(out_sizes.back()); + int32_t orig_N = + utils::safe_downcast(out_sizes.at(out_sizes.size() - 1)); ValueRef orig_N_ref = graph.add_scalar(static_cast(orig_N)); ValueRef has_bias_ref = graph.add_scalar(has_bias); @@ -109,79 +108,4 @@ void add_linear_coopmat_node( resize_linear_node)); } -// ── Matmul coopmat ── - -static vkapi::ShaderInfo pick_matmul_coopmat_shader( - ComputeGraph* graph, - const std::vector& args, - const std::vector& resize_args) { - (void)resize_args; - const ValueRef out = args.at(0).refs.at(0); - std::string kernel_name = "matmul_coopmat"; - kernel_name.reserve(kShaderNameReserve); - add_dtype_suffix(kernel_name, graph->dtype_of(out)); - return VK_KERNEL_FROM_STR(kernel_name); -} - -static GlobalWorkGrid pick_matmul_coopmat_gwg( - ComputeGraph* graph, - const vkapi::ShaderInfo& shader, - const std::vector& args, - const std::vector& resize_args) { - (void)shader; - (void)resize_args; - const ValueRef out = args.at(0).refs.at(0); - const auto out_sizes = graph->sizes_of(out); - uint32_t M = out_sizes.at(out_sizes.size() - 2); - uint32_t N = out_sizes.at(out_sizes.size() - 1); - uint32_t num_tiles_n = utils::div_up(N, kCoopmatTileN); - uint32_t num_tiles_m = utils::div_up(M, kCoopmatTileM); - // Each workgroup processes one WG_TILE_M x WG_TILE_N output tile via - // cooperative-matrix MMAs across its 4 subgroups. We want the dispatch - // to launch exactly num_tiles_n x num_tiles_m workgroups. - // - // The framework computes the group count as - // group_count = div_up(gwg, lwg) - // (see Context.cpp + Command.cpp). With lwg = (kCoopmatInvocations, - // 1, 1), multiplying num_tiles_n by kCoopmatInvocations cancels the - // div, yielding group_count.x = num_tiles_n. - return GlobalWorkGrid( - {num_tiles_n * kCoopmatInvocations, num_tiles_m, 1u}, - kTiledWorkGrid, - LocalWorkGroup(kCoopmatInvocations, 1u, 1u)); -} - -void add_matmul_coopmat_node( - ComputeGraph& graph, - const ValueRef mat1, - const ValueRef mat2, - const ValueRef out) { - VK_CHECK_COND(graph.packed_dim_of(mat1) == WHCN::kWidthDim); - VK_CHECK_COND(graph.packed_dim_of(mat2) == WHCN::kWidthDim); - VK_CHECK_COND(graph.packed_dim_of(out) == WHCN::kWidthDim); - VK_CHECK_COND( - graph.storage_type_of(out) == utils::kBuffer, - "matmul_coopmat requires buffer storage"); - - ValueRef has_bias_ref = graph.add_scalar(false); - - graph.execute_nodes().emplace_back(new DynamicDispatchNode( - graph, - pick_matmul_coopmat_shader, - pick_matmul_coopmat_gwg, - pick_required_lwg, - // Inputs and Outputs — same binding order as matmul_vec - {{out, vkapi::kWrite}, {{mat1, mat2}, vkapi::kRead}}, - // Shader params buffers — same UBOs as matmul_vec - {graph.sizes_ubo(mat1), graph.sizes_ubo(mat2)}, - // Push Constants - {}, - // Specialization Constants - {}, - // Resize Args - {has_bias_ref}, - // Resizing Logic - resize_matmul_tiled_node)); -} - } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/LinearCoopmat.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/LinearCoopmat.h new file mode 100644 index 00000000000..e5aac00c04e --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/LinearCoopmat.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +void add_linear_coopmat_node( + ComputeGraph& graph, + const ValueRef input, + const ValueRef packed_weight, + const ValueRef packed_bias, + bool has_bias, + const ValueRef out, + int32_t weight_B = 1); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/LinearDw.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/backward/LinearDw.cpp similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/LinearDw.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/backward/LinearDw.cpp diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taLinear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinear.cpp similarity index 97% rename from backends/vulkan/runtime/graph/ops/impl/Q8taLinear.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinear.cpp index ea1a87b910d..fba34214dab 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taLinear.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinear.cpp @@ -6,14 +6,14 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include -#include #include #include +#include #include namespace vkcompute { diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taLinear.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinear.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Q8taLinear.h rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinear.h diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinearGemv.cpp similarity index 97% rename from backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinearGemv.cpp index a3944641839..2861bb1d7b9 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinearGemv.cpp @@ -6,14 +6,14 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include -#include #include #include +#include #include namespace vkcompute { diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinearGemv.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.h rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinearGemv.h diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/QuantizedLinear.cpp similarity index 63% rename from backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/QuantizedLinear.cpp index 3f1ea14f20b..3b3e11c8498 100644 --- a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/QuantizedLinear.cpp @@ -6,13 +6,13 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include -#include #include -#include #include +#include +#include #include namespace vkcompute { @@ -51,26 +51,6 @@ void resize_linear_qw_node( graph->virtual_resize(output, new_out_sizes); } -// Per-shader coopmat tile geometry (must match each shader's yaml). -// Workgroup size (wg_size) = SG_GRID_X * SG_GRID_Y * SUBGROUP_SIZE. -// linear_q4gsw_coopmat 128x64x16, 2x2 subgroups x 32 (forced) -> 128 -// linear_dq8ca_q4gsw_coopmat 128x64x32, 2x2 subgroups x 64 -> 256 -// (The int8-MMA shaders stay on wave64: int8 WMMA at forced subgroup 32 -// crashes the Xclipse PAL compiler.) -struct CoopmatTileDims { - uint32_t m; - uint32_t n; - uint32_t k; - // Threads per workgroup = SG_GRID_X * SG_GRID_Y * SUBGROUP_SIZE. MUST match - // the WG_SIZE the shader yaml resolves to, or the launched thread count won't - // match the shader's staging passes (out-of-bounds). - uint32_t wg_size; -}; -// linear_qw_coopmat.yaml: 128x64, 2x2 subgroup grid, sg32 -> WG_SIZE 128. -constexpr CoopmatTileDims kQ4gswCoopmatDims = {128, 64, 16, 128}; -// linear_dq8ca_qw_coopmat.yaml: 128x64, 2x2 grid, sg64 -> WG_SIZE 256. -constexpr CoopmatTileDims kDq8caQ4gswCoopmatDims = {128, 64, 32, 256}; - static CoopmatTileDims coopmat_tile_dims(const std::string& kernel_name) { // Exact prefix matches (the "linear_dq8ca_*" names must not match the // weight-only entries). @@ -88,16 +68,19 @@ GlobalWorkGrid quantized_linear_gwg( const vkapi::ShaderInfo& shader, const std::vector& args, const std::vector& resize_args) { + (void)resize_args; const ValueRef out = args.at(0).refs.at(0); std::vector out_sizes = graph->sizes_of(out); // width - const uint32_t N = utils::val_at(-1, out_sizes); + const uint32_t N = + utils::safe_downcast(utils::val_at(-1, out_sizes)); // height - const uint32_t M = utils::val_at(-2, out_sizes); + const uint32_t M = + utils::safe_downcast(utils::val_at(-2, out_sizes)); // Coopmat variants dispatch a 256-thread WG per 64x64 output tile. Mirrors - // GemmCoopmat.cpp's pick_linear_coopmat_gwg — the multiplication + // LinearCoopmat.cpp's pick_linear_coopmat_gwg — the multiplication // by kCoopmatInvocations cancels the framework's div_up, since // lwg = {256, 1, 1}. if (shader.kernel_name.find("_coopmat") != std::string::npos) { @@ -159,15 +142,15 @@ LocalWorkGroup quantized_linear_lwg( // (M, N, K, dtype, output_storage, group_size) tuple. Preconditions match what // linear_q4gsw_coopmat.glsl assumes; the subgroup_size == 64 check scopes this // to wave64 devices (e.g. AMD RDNA), which the coopmat tiling is tuned for. -static bool can_use_q4gsw_coopmat( +bool can_use_q4gsw_coopmat( ComputeGraph* graph, const ValueRef output, const ValueRef fp_input, int64_t group_size, const ValueRef bias, - int64_t tile_m = kCoopmatTileM, - int64_t tile_n = kCoopmatTileN, - int64_t tile_k = kCoopmatTileK) { + int64_t tile_m, + int64_t tile_n, + int64_t tile_k) { // The coopmat shaders only build HAS_BIAS=false variants, so they would // silently drop a bias. Fall back to the tiled path (which applies bias at // runtime via the apply_bias spec constant) whenever a bias is present. @@ -288,56 +271,6 @@ vkapi::ShaderInfo pick_linear_qw_shader( return VK_KERNEL_FROM_STR(kernel_name); } -vkapi::ShaderInfo pick_linear_dqa_qw_shader( - ComputeGraph* graph, - const std::vector& args, - const std::vector& resize_args) { - (void)resize_args; - - const ValueRef out = args.at(0).refs.at(0); - const ValueRef fp_input = args.at(1).refs.at(0); - const ValueRef int_input = args.at(1).refs.at(1); - (void)int_input; - const ValueRef input_zp = args.at(1).refs.at(4); - const ValueRef int_weight = args.at(1).refs.at(5); - - const bool weight_is_4bit = resize_args.at(0) != kDummyValueRef; - const bool is_gemv_case = is_gemv(graph, fp_input); - - // Use the coopmat shader for 4-bit dq8ca dispatches when the device - // enumerates VK_COMPONENT_TYPE_SINT8_KHR in its cooperative matrix property - // list and the shape aligns; tiled otherwise. - if (weight_is_4bit && !is_gemv_case && - graph->context()->adapter_ptr()->supports_int8_cooperative_matrix()) { - const int64_t group_size = - graph->extract_scalar(resize_args.at(0)); - if (can_use_q4gsw_coopmat( - graph, - out, - fp_input, - group_size, - resize_args.at(2), - kDq8caQ4gswCoopmatDims.m, - kDq8caQ4gswCoopmatDims.n, - kDq8caQ4gswCoopmatDims.k)) { - std::string kernel_name = "linear_dq8ca_q4gsw_coopmat"; - add_storage_type_suffix(kernel_name, graph->storage_type_of(out)); - add_storage_type_suffix(kernel_name, graph->storage_type_of(int_weight)); - add_dtype_suffix(kernel_name, graph->dtype_of(out)); - return VK_KERNEL_FROM_STR(kernel_name); - } - } - - std::string kernel_name = "linear_dq8ca_q4gsw"; - kernel_name += is_gemv_case ? "_coop" : "_tiled"; - add_storage_type_suffix(kernel_name, graph->storage_type_of(out)); - add_storage_type_suffix(kernel_name, graph->storage_type_of(int_weight)); - add_dtype_suffix(kernel_name, graph->dtype_of(out)); - add_zp_dtype_mode_suffix(kernel_name, graph->dtype_of(input_zp)); - - return VK_KERNEL_FROM_STR(kernel_name); -} - // // Prepacking nodes // @@ -500,6 +433,7 @@ void add_linear_qw_node( const ValueRef bias_data, const ValueRef packed_bias, const ValueRef output) { + (void)packed_weight_zeros; // Only certain quantization types supported at the moment VK_CHECK_COND( weight_quant_config.granularity == kPerChannel || @@ -556,160 +490,6 @@ void add_linear_qw_node( resize_linear_qw_node)); } -void add_linear_qa_qw_node( - ComputeGraph& graph, - const QuantizationConfig& input_quant_config, - const QuantizationConfig& weight_quant_config, - const ValueRef fp_input, - const ValueRef packed_int_input, - const ValueRef packed_input_scale, - const ValueRef packed_input_zp, - const ValueRef input_scale_data, - const ValueRef input_zp_data, - const ValueRef weight_data, - const ValueRef packed_weight, - const ValueRef packed_weight_sums, - const ValueRef packed_weight_scales, - const ValueRef group_size, - const ValueRef bias_data, - const ValueRef packed_bias, - const ValueRef output) { - VK_CHECK_COND(input_quant_config.granularity == kPerTensor); - VK_CHECK_COND(input_quant_config.nbits == 8); - VK_CHECK_COND(weight_quant_config.granularity == kPerChannel); - VK_CHECK_COND(weight_quant_config.is_symmetric); - VK_CHECK_COND(weight_quant_config.nbits == 8); - - float scale = graph.extract_scalar(input_scale_data); - int32_t zp = graph.extract_scalar(input_zp_data); - - // Get shader for quantized linear - std::string kernel_name = "linear_q8ta_q8csw_tiled"; - add_storage_type_suffix(kernel_name, graph.storage_type_of(output)); - add_storage_type_suffix(kernel_name, graph.storage_type_of(packed_int_input)); - add_storage_type_suffix(kernel_name, graph.storage_type_of(packed_weight)); - add_dtype_suffix(kernel_name, graph.dtype_of(output)); - vkapi::ShaderInfo shader = VK_KERNEL_FROM_STR(kernel_name); - - vkapi::ParamsBindList param_buffers = { - graph.sizes_ubo(output), graph.sizes_ubo(packed_int_input)}; - - std::vector push_constants = { - PushConstantDataInfo(&scale, sizeof(scale)), - PushConstantDataInfo(&zp, sizeof(zp)), - }; - - uint32_t apply_bias = 1; - if (graph.val_is_none(bias_data)) { - apply_bias = 0; - } - - graph.execute_nodes().emplace_back(new DynamicDispatchNode( - graph, - VK_KERNEL_FROM_STR(kernel_name), - quantized_linear_gwg, - quantized_linear_lwg, - // Inputs and Outputs - {{output, vkapi::kWrite}, - {{packed_int_input, - packed_weight, - packed_weight_sums, - packed_weight_scales, - packed_bias}, - vkapi::kRead}}, - // Shader params buffers - param_buffers, - // Push Constants - push_constants, - // Specialization Constants - {apply_bias}, - // Resize args - {fp_input}, - // Resizing Logic - nullptr)); -} - -void add_linear_dqa_qw_node( - ComputeGraph& graph, - const QuantizationConfig& input_quant_config, - const QuantizationConfig& weight_quant_config, - const ValueRef fp_input, - const ValueRef packed_int_input, - const ValueRef int_input_sums, - const ValueRef packed_input_scale, - const ValueRef packed_input_zp, - const ValueRef input_scale_data, - const ValueRef input_zp_data, - const ValueRef weight_data, - const ValueRef packed_weight, - const ValueRef packed_weight_sums, - const ValueRef packed_weight_scales, - const ValueRef group_size, - const ValueRef bias_data, - const ValueRef packed_bias, - const ValueRef output) { - VK_CHECK_COND(input_quant_config.granularity == kPerChannel); - VK_CHECK_COND(input_quant_config.nbits == 8); - VK_CHECK_COND(input_quant_config.is_dynamic); - - VK_CHECK_COND(weight_quant_config.granularity == kPerGroup); - VK_CHECK_COND(weight_quant_config.is_symmetric); - VK_CHECK_COND(weight_quant_config.nbits == 4); - - vkapi::ParamsBindList param_buffers = { - graph.sizes_ubo(output), graph.sizes_ubo(fp_input)}; - - uint32_t apply_bias = 1; - if (graph.val_is_none(bias_data)) { - apply_bias = 0; - } - - int32_t K4_per_group = 0; - int32_t coopmat_k_iters = 0; - const int32_t K_dim = graph.size_at(-1, fp_input); - if (weight_quant_config.nbits == 4) { - int32_t group_size_val = graph.extract_scalar(group_size); - K4_per_group = utils::div_up(group_size_val, int32_t(4)); - coopmat_k_iters = K_dim / group_size_val; - } - - const ValueRef is_4bit_flag = - weight_quant_config.nbits == 4 ? group_size : kDummyValueRef; - - graph.execute_nodes().emplace_back(new DynamicDispatchNode( - graph, - pick_linear_dqa_qw_shader, - quantized_linear_gwg, - quantized_linear_lwg, - // Inputs and Outputs - {{output, vkapi::kWrite}, - {{fp_input, - packed_int_input, - int_input_sums, - packed_input_scale, - packed_input_zp, - packed_weight, - packed_weight_sums, - packed_weight_scales, - packed_bias}, - vkapi::kRead}}, - // Shader params buffers - param_buffers, - // Push Constants - {}, - // Specialization Constants - // 4th spec const: output width N for coopMatStore (see - // add_linear_qw_node). - {apply_bias, - K4_per_group, - coopmat_k_iters, - graph.size_at(-1, output)}, - // Resize args (resize_args.at(2) = bias_data, read by the coopmat gate) - {is_4bit_flag, weight_data, bias_data}, - // Resizing Logic - resize_linear_qw_node)); -} - // // High level operator impl // @@ -728,6 +508,7 @@ void quantized_linear_impl( const ValueRef group_size, const ValueRef bias_data, const ValueRef output) { + (void)weight_zeros_data; std::vector input_sizes = graph.sizes_of(fp_input); std::vector weight_sizes = graph.sizes_of(weight_data); @@ -898,141 +679,4 @@ void quantized_linear_impl( output); } -void linear_q8ta_q8csw(ComputeGraph& graph, const std::vector& args) { - int32_t idx = 0; - const ValueRef fp_input = args.at(idx++); - const ValueRef input_scale = args.at(idx++); - const ValueRef input_zp = args.at(idx++); - const ValueRef weight_data = args.at(idx++); - const ValueRef weight_sums_data = args.at(idx++); - const ValueRef weight_scales_data = args.at(idx++); - const ValueRef bias_data = args.at(idx++); - const ValueRef output = args.at(idx++); - - const int64_t K = graph.size_at(-1, fp_input); - - QuantizationConfig input_quant_config(8, kPerTensor, {}, false); - QuantizationConfig weight_quant_config(8, kPerChannel, {K}); - - quantized_linear_impl( - graph, - input_quant_config, - weight_quant_config, - fp_input, - input_scale, - input_zp, - weight_data, - weight_sums_data, - weight_scales_data, - kDummyValueRef, // weight_zeros_data - kDummyValueRef, // group_size - bias_data, - output); -} - -void linear_q8csw(ComputeGraph& graph, const std::vector& args) { - int32_t idx = 0; - const ValueRef fp_input = args.at(idx++); - const ValueRef weight_data = args.at(idx++); - const ValueRef weight_scales_data = args.at(idx++); - const ValueRef bias_data = args.at(idx++); - const ValueRef output = args.at(idx++); - - const int64_t K = graph.size_at(-1, fp_input); - - QuantizationConfig input_quant_config(32, kNoQuantization, {}); - QuantizationConfig weight_quant_config(8, kPerChannel, {K}); - - quantized_linear_impl( - graph, - input_quant_config, - weight_quant_config, - fp_input, - kDummyValueRef, // input scale - kDummyValueRef, // input zp - weight_data, - kDummyValueRef, // weight sums - weight_scales_data, - kDummyValueRef, // weight zeros - kDummyValueRef, // group size - bias_data, - output); -} - -// aten._weight_int8pack_mm is what the AOT weight-only int8 fusion -// (FuseQuantizedOpsTransform) emits. It carries the same operands as -// et_vk.linear_q8csw minus the bias, so it runs through the same -// implementation. -void weight_int8pack_mm( - ComputeGraph& graph, - const std::vector& args) { - int32_t idx = 0; - const ValueRef fp_input = args.at(idx++); - const ValueRef weight_data = args.at(idx++); - const ValueRef weight_scales_data = args.at(idx++); - const ValueRef output = args.at(idx++); - - const int64_t K = graph.size_at(-1, fp_input); - - QuantizationConfig input_quant_config(32, kNoQuantization, {}); - QuantizationConfig weight_quant_config(8, kPerChannel, {K}); - - quantized_linear_impl( - graph, - input_quant_config, - weight_quant_config, - fp_input, - kDummyValueRef, // input scale - kDummyValueRef, // input zp - weight_data, - kDummyValueRef, // weight sums - weight_scales_data, - kDummyValueRef, // weight zeros - kDummyValueRef, // group size - kDummyValueRef, // bias - output); -} - -void linear_dq8ca_q4gsw( - ComputeGraph& graph, - const std::vector& args) { - int32_t idx = 0; - const ValueRef fp_input = args.at(idx++); - const ValueRef input_scale = args.at(idx++); - const ValueRef input_zp = args.at(idx++); - const ValueRef weight_data = args.at(idx++); - const ValueRef weight_sums_data = args.at(idx++); - const ValueRef weight_scales_data = args.at(idx++); - const ValueRef group_size = args.at(idx++); - const ValueRef bias_data = args.at(idx++); - const ValueRef output = args.at(idx++); - - const int64_t group_size_val = graph.extract_scalar(group_size); - - QuantizationConfig input_quant_config(8, kPerChannel, {}, false, true); - QuantizationConfig weight_quant_config(4, kPerGroup, {group_size_val}); - - quantized_linear_impl( - graph, - input_quant_config, - weight_quant_config, - fp_input, - input_scale, - input_zp, - weight_data, - weight_sums_data, - weight_scales_data, - kDummyValueRef, // weight_zeros_data - group_size, // group_size - bias_data, - output); -} - -REGISTER_OPERATORS { - VK_REGISTER_OP(et_vk.linear_q8ta_q8csw.default, linear_q8ta_q8csw); - VK_REGISTER_OP(et_vk.linear_q8csw.default, linear_q8csw); - VK_REGISTER_OP(aten._weight_int8pack_mm.default, weight_int8pack_mm); - VK_REGISTER_OP(et_vk.linear_dq8ca_q4gsw.default, linear_dq8ca_q4gsw); -} - } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/QuantizedLinear.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/QuantizedLinear.h new file mode 100644 index 00000000000..23f70416754 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/QuantizedLinear.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include +#include +#include + +namespace vkcompute { + +// Per-shader coopmat tile geometry (must match each shader's yaml). +// Workgroup size (wg_size) = SG_GRID_X * SG_GRID_Y * SUBGROUP_SIZE. +// linear_q4gsw_coopmat 128x64x16, 2x2 subgroups x 32 (forced) -> 128 +// linear_dq8ca_q4gsw_coopmat 128x64x32, 2x2 subgroups x 64 -> 256 +// (The int8-MMA shaders stay on wave64: int8 WMMA at forced subgroup 32 +// crashes the Xclipse PAL compiler.) +struct CoopmatTileDims { + uint32_t m; + uint32_t n; + uint32_t k; + // Threads per workgroup = SG_GRID_X * SG_GRID_Y * SUBGROUP_SIZE. MUST match + // the WG_SIZE the shader yaml resolves to, or the launched thread count won't + // match the shader's staging passes (out-of-bounds). + uint32_t wg_size; +}; +// linear_qw_coopmat.yaml: 128x64, 2x2 subgroup grid, sg32 -> WG_SIZE 128. +constexpr CoopmatTileDims kQ4gswCoopmatDims = {128, 64, 16, 128}; +// linear_dq8ca_qw_coopmat.yaml: 128x64, 2x2 grid, sg64 -> WG_SIZE 256. +constexpr CoopmatTileDims kDq8caQ4gswCoopmatDims = {128, 64, 32, 256}; + +void resize_linear_qw_node( + ComputeGraph* graph, + const std::vector& args, + const std::vector& extra_args); + +GlobalWorkGrid quantized_linear_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args); + +LocalWorkGroup quantized_linear_lwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args, + const std::vector& resize_args); + +bool can_use_q4gsw_coopmat( + ComputeGraph* graph, + const ValueRef output, + const ValueRef fp_input, + int64_t group_size, + const ValueRef bias, + int64_t tile_m = kCoopmatTileM, + int64_t tile_n = kCoopmatTileN, + int64_t tile_k = kCoopmatTileK); + +ValueRef prepack_quantized_linear_weight( + ComputeGraph& graph, + const QuantizationConfig& weight_quant_config, + const ValueRef qmat2_data, + const bool use_unsigned_dot = false); + +void quantized_linear_impl( + ComputeGraph& graph, + const QuantizationConfig& input_quant_config, + const QuantizationConfig& weight_quant_config, + const ValueRef fp_input, + const ValueRef input_scale, + const ValueRef input_zp, + const ValueRef weight_data, + const ValueRef weight_sums_data, + const ValueRef weight_scales_data, + const ValueRef weight_zeros_data, + const ValueRef group_size, + const ValueRef bias_data, + const ValueRef output); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.cpp new file mode 100644 index 00000000000..dd13ec2ebb5 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.cpp @@ -0,0 +1,192 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include + +#include +#include +#include +#include + +namespace vkcompute { + +vkapi::ShaderInfo pick_linear_dqa_qw_shader( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + + const ValueRef out = args.at(0).refs.at(0); + const ValueRef fp_input = args.at(1).refs.at(0); + const ValueRef int_input = args.at(1).refs.at(1); + (void)int_input; + const ValueRef input_zp = args.at(1).refs.at(4); + const ValueRef int_weight = args.at(1).refs.at(5); + + const bool weight_is_4bit = resize_args.at(0) != kDummyValueRef; + const bool is_gemv_case = is_gemv(graph, fp_input); + + // Use the coopmat shader for 4-bit dq8ca dispatches when the device + // enumerates VK_COMPONENT_TYPE_SINT8_KHR in its cooperative matrix property + // list and the shape aligns; tiled otherwise. + if (weight_is_4bit && !is_gemv_case && + graph->context()->adapter_ptr()->supports_int8_cooperative_matrix()) { + const int64_t group_size = + graph->extract_scalar(resize_args.at(0)); + if (can_use_q4gsw_coopmat( + graph, + out, + fp_input, + group_size, + resize_args.at(2), + kDq8caQ4gswCoopmatDims.m, + kDq8caQ4gswCoopmatDims.n, + kDq8caQ4gswCoopmatDims.k)) { + std::string kernel_name = "linear_dq8ca_q4gsw_coopmat"; + add_storage_type_suffix(kernel_name, graph->storage_type_of(out)); + add_storage_type_suffix(kernel_name, graph->storage_type_of(int_weight)); + add_dtype_suffix(kernel_name, graph->dtype_of(out)); + return VK_KERNEL_FROM_STR(kernel_name); + } + } + + std::string kernel_name = "linear_dq8ca_q4gsw"; + kernel_name += is_gemv_case ? "_coop" : "_tiled"; + add_storage_type_suffix(kernel_name, graph->storage_type_of(out)); + add_storage_type_suffix(kernel_name, graph->storage_type_of(int_weight)); + add_dtype_suffix(kernel_name, graph->dtype_of(out)); + add_zp_dtype_mode_suffix(kernel_name, graph->dtype_of(input_zp)); + + return VK_KERNEL_FROM_STR(kernel_name); +} + +void add_linear_dqa_qw_node( + ComputeGraph& graph, + const QuantizationConfig& input_quant_config, + const QuantizationConfig& weight_quant_config, + const ValueRef fp_input, + const ValueRef packed_int_input, + const ValueRef int_input_sums, + const ValueRef packed_input_scale, + const ValueRef packed_input_zp, + const ValueRef input_scale_data, + const ValueRef input_zp_data, + const ValueRef weight_data, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef group_size, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef output) { + (void)input_scale_data; + (void)input_zp_data; + VK_CHECK_COND(input_quant_config.granularity == kPerChannel); + VK_CHECK_COND(input_quant_config.nbits == 8); + VK_CHECK_COND(input_quant_config.is_dynamic); + + VK_CHECK_COND(weight_quant_config.granularity == kPerGroup); + VK_CHECK_COND(weight_quant_config.is_symmetric); + VK_CHECK_COND(weight_quant_config.nbits == 4); + + vkapi::ParamsBindList param_buffers = { + graph.sizes_ubo(output), graph.sizes_ubo(fp_input)}; + + uint32_t apply_bias = 1; + if (graph.val_is_none(bias_data)) { + apply_bias = 0; + } + + int32_t K4_per_group = 0; + int32_t coopmat_k_iters = 0; + const int32_t K_dim = graph.size_at(-1, fp_input); + if (weight_quant_config.nbits == 4) { + int32_t group_size_val = graph.extract_scalar(group_size); + K4_per_group = utils::div_up(group_size_val, int32_t(4)); + coopmat_k_iters = K_dim / group_size_val; + } + + const ValueRef is_4bit_flag = + weight_quant_config.nbits == 4 ? group_size : kDummyValueRef; + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + pick_linear_dqa_qw_shader, + quantized_linear_gwg, + quantized_linear_lwg, + // Inputs and Outputs + {{output, vkapi::kWrite}, + {{fp_input, + packed_int_input, + int_input_sums, + packed_input_scale, + packed_input_zp, + packed_weight, + packed_weight_sums, + packed_weight_scales, + packed_bias}, + vkapi::kRead}}, + // Shader params buffers + param_buffers, + // Push Constants + {}, + // Specialization Constants + // 4th spec const: output width N for coopMatStore (see + // add_linear_qw_node). + {apply_bias, + K4_per_group, + coopmat_k_iters, + graph.size_at(-1, output)}, + // Resize args (resize_args.at(2) = bias_data, read by the coopmat gate) + {is_4bit_flag, weight_data, bias_data}, + // Resizing Logic + resize_linear_qw_node)); +} + +void linear_dq8ca_q4gsw( + ComputeGraph& graph, + const std::vector& args) { + int32_t idx = 0; + const ValueRef fp_input = args.at(idx++); + const ValueRef input_scale = args.at(idx++); + const ValueRef input_zp = args.at(idx++); + const ValueRef weight_data = args.at(idx++); + const ValueRef weight_sums_data = args.at(idx++); + const ValueRef weight_scales_data = args.at(idx++); + const ValueRef group_size = args.at(idx++); + const ValueRef bias_data = args.at(idx++); + const ValueRef output = args.at(idx++); + + const int64_t group_size_val = graph.extract_scalar(group_size); + + QuantizationConfig input_quant_config(8, kPerChannel, {}, false, true); + QuantizationConfig weight_quant_config(4, kPerGroup, {group_size_val}); + + quantized_linear_impl( + graph, + input_quant_config, + weight_quant_config, + fp_input, + input_scale, + input_zp, + weight_data, + weight_sums_data, + weight_scales_data, + kDummyValueRef, // weight_zeros_data + group_size, // group_size + bias_data, + output); +} + +REGISTER_OPERATORS { + VK_REGISTER_OP(et_vk.linear_dq8ca_q4gsw.default, linear_dq8ca_q4gsw); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.h new file mode 100644 index 00000000000..0cff158a323 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +namespace vkcompute { + +void add_linear_dqa_qw_node( + ComputeGraph& graph, + const QuantizationConfig& input_quant_config, + const QuantizationConfig& weight_quant_config, + const ValueRef fp_input, + const ValueRef packed_int_input, + const ValueRef int_input_sums, + const ValueRef packed_input_scale, + const ValueRef packed_input_zp, + const ValueRef input_scale_data, + const ValueRef input_zp_data, + const ValueRef weight_data, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef group_size, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef output); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.cpp new file mode 100644 index 00000000000..c6c69be6a1c --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.cpp @@ -0,0 +1,128 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include + +#include +#include +#include + +namespace vkcompute { + +void add_linear_qa_qw_node( + ComputeGraph& graph, + const QuantizationConfig& input_quant_config, + const QuantizationConfig& weight_quant_config, + const ValueRef fp_input, + const ValueRef packed_int_input, + const ValueRef packed_input_scale, + const ValueRef packed_input_zp, + const ValueRef input_scale_data, + const ValueRef input_zp_data, + const ValueRef weight_data, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef group_size, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef output) { + VK_CHECK_COND(input_quant_config.granularity == kPerTensor); + VK_CHECK_COND(input_quant_config.nbits == 8); + VK_CHECK_COND(weight_quant_config.granularity == kPerChannel); + VK_CHECK_COND(weight_quant_config.is_symmetric); + VK_CHECK_COND(weight_quant_config.nbits == 8); + + float scale = graph.extract_scalar(input_scale_data); + int32_t zp = graph.extract_scalar(input_zp_data); + + // Get shader for quantized linear + std::string kernel_name = "linear_q8ta_q8csw_tiled"; + add_storage_type_suffix(kernel_name, graph.storage_type_of(output)); + add_storage_type_suffix(kernel_name, graph.storage_type_of(packed_int_input)); + add_storage_type_suffix(kernel_name, graph.storage_type_of(packed_weight)); + add_dtype_suffix(kernel_name, graph.dtype_of(output)); + vkapi::ShaderInfo shader = VK_KERNEL_FROM_STR(kernel_name); + + vkapi::ParamsBindList param_buffers = { + graph.sizes_ubo(output), graph.sizes_ubo(packed_int_input)}; + + std::vector push_constants = { + PushConstantDataInfo(&scale, sizeof(scale)), + PushConstantDataInfo(&zp, sizeof(zp)), + }; + + uint32_t apply_bias = 1; + if (graph.val_is_none(bias_data)) { + apply_bias = 0; + } + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + VK_KERNEL_FROM_STR(kernel_name), + quantized_linear_gwg, + quantized_linear_lwg, + // Inputs and Outputs + {{output, vkapi::kWrite}, + {{packed_int_input, + packed_weight, + packed_weight_sums, + packed_weight_scales, + packed_bias}, + vkapi::kRead}}, + // Shader params buffers + param_buffers, + // Push Constants + push_constants, + // Specialization Constants + {apply_bias}, + // Resize args + {fp_input}, + // Resizing Logic + nullptr)); +} + +void linear_q8ta_q8csw(ComputeGraph& graph, const std::vector& args) { + int32_t idx = 0; + const ValueRef fp_input = args.at(idx++); + const ValueRef input_scale = args.at(idx++); + const ValueRef input_zp = args.at(idx++); + const ValueRef weight_data = args.at(idx++); + const ValueRef weight_sums_data = args.at(idx++); + const ValueRef weight_scales_data = args.at(idx++); + const ValueRef bias_data = args.at(idx++); + const ValueRef output = args.at(idx++); + + const int64_t K = graph.size_at(-1, fp_input); + + QuantizationConfig input_quant_config(8, kPerTensor, {}, false); + QuantizationConfig weight_quant_config(8, kPerChannel, {K}); + + quantized_linear_impl( + graph, + input_quant_config, + weight_quant_config, + fp_input, + input_scale, + input_zp, + weight_data, + weight_sums_data, + weight_scales_data, + kDummyValueRef, // weight_zeros_data + kDummyValueRef, // group_size + bias_data, + output); +} + +REGISTER_OPERATORS { + VK_REGISTER_OP(et_vk.linear_q8ta_q8csw.default, linear_q8ta_q8csw); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.h new file mode 100644 index 00000000000..661da132b8c --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +namespace vkcompute { + +void add_linear_qa_qw_node( + ComputeGraph& graph, + const QuantizationConfig& input_quant_config, + const QuantizationConfig& weight_quant_config, + const ValueRef fp_input, + const ValueRef packed_int_input, + const ValueRef packed_input_scale, + const ValueRef packed_input_zp, + const ValueRef input_scale_data, + const ValueRef input_zp_data, + const ValueRef weight_data, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef group_size, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef output); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinearQCSNW.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/QuantizedLinearQCSNW.cpp similarity index 99% rename from backends/vulkan/runtime/graph/ops/impl/QuantizedLinearQCSNW.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/QuantizedLinearQCSNW.cpp index c2e45b96418..99b0fc1e225 100644 --- a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinearQCSNW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/QuantizedLinearQCSNW.cpp @@ -298,7 +298,7 @@ void linear_qcs4w(ComputeGraph& graph, const std::vector& args) { } REGISTER_OPERATORS { - // aten._weight_int8pack_mm is registered in QuantizedLinear.cpp, on the + // aten._weight_int8pack_mm is registered in Q8cswLinear.cpp, on the // maintained weight-only quantized linear implementation. The 8-bit path // here dispatched to linear_qcs8w_* shaders that no longer exist. VK_REGISTER_OP(et_vk.linear_qcs4w.default, linear_qcs4w); diff --git a/backends/vulkan/runtime/graph/ops/impl/Q4gswLinear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/Q4gswLinear.cpp similarity index 99% rename from backends/vulkan/runtime/graph/ops/impl/Q4gswLinear.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/Q4gswLinear.cpp index cf8afc8c8c1..8e7d16092c7 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q4gswLinear.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/Q4gswLinear.cpp @@ -6,7 +6,7 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/Q4gswLinear.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/Q4gswLinear.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Q4gswLinear.h rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/Q4gswLinear.h diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinearBackward.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/backward/QuantizedLinearBackward.cpp similarity index 97% rename from backends/vulkan/runtime/graph/ops/impl/QuantizedLinearBackward.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/backward/QuantizedLinearBackward.cpp index 8bfe1681bbf..01ebb2f692d 100644 --- a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinearBackward.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/backward/QuantizedLinearBackward.cpp @@ -11,7 +11,7 @@ #include #include -#include +#include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinearRequant.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/backward/QuantizedLinearRequant.cpp similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/QuantizedLinearRequant.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/backward/QuantizedLinearRequant.cpp diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q8csw/Q8cswLinear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q8csw/Q8cswLinear.cpp new file mode 100644 index 00000000000..89657534b9b --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q8csw/Q8cswLinear.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include + +namespace vkcompute { + +void linear_q8csw(ComputeGraph& graph, const std::vector& args) { + int32_t idx = 0; + const ValueRef fp_input = args.at(idx++); + const ValueRef weight_data = args.at(idx++); + const ValueRef weight_scales_data = args.at(idx++); + const ValueRef bias_data = args.at(idx++); + const ValueRef output = args.at(idx++); + + const int64_t K = graph.size_at(-1, fp_input); + + QuantizationConfig input_quant_config(32, kNoQuantization, {}); + QuantizationConfig weight_quant_config(8, kPerChannel, {K}); + + quantized_linear_impl( + graph, + input_quant_config, + weight_quant_config, + fp_input, + kDummyValueRef, // input scale + kDummyValueRef, // input zp + weight_data, + kDummyValueRef, // weight sums + weight_scales_data, + kDummyValueRef, // weight zeros + kDummyValueRef, // group size + bias_data, + output); +} + +// aten._weight_int8pack_mm is what the AOT weight-only int8 fusion +// (FuseQuantizedOpsTransform) emits. It carries the same operands as +// et_vk.linear_q8csw minus the bias, so it runs through the same +// implementation. +void weight_int8pack_mm( + ComputeGraph& graph, + const std::vector& args) { + int32_t idx = 0; + const ValueRef fp_input = args.at(idx++); + const ValueRef weight_data = args.at(idx++); + const ValueRef weight_scales_data = args.at(idx++); + const ValueRef output = args.at(idx++); + + const int64_t K = graph.size_at(-1, fp_input); + + QuantizationConfig input_quant_config(32, kNoQuantization, {}); + QuantizationConfig weight_quant_config(8, kPerChannel, {K}); + + quantized_linear_impl( + graph, + input_quant_config, + weight_quant_config, + fp_input, + kDummyValueRef, // input scale + kDummyValueRef, // input zp + weight_data, + kDummyValueRef, // weight sums + weight_scales_data, + kDummyValueRef, // weight zeros + kDummyValueRef, // group size + kDummyValueRef, // bias + output); +} + +REGISTER_OPERATORS { + VK_REGISTER_OP(et_vk.linear_q8csw.default, linear_q8csw); + VK_REGISTER_OP(aten._weight_int8pack_mm.default, weight_int8pack_mm); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Matmul.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatMul.cpp similarity index 95% rename from backends/vulkan/runtime/graph/ops/impl/Matmul.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatMul.cpp index fc86255be51..e314436cc7c 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Matmul.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatMul.cpp @@ -9,11 +9,12 @@ #include #include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/MatMul.h b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatMul.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/MatMul.h rename to backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatMul.h diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.cpp new file mode 100644 index 00000000000..1cc9730ac9b --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include + +#include + +namespace vkcompute { + +static vkapi::ShaderInfo pick_matmul_coopmat_shader( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + std::string kernel_name = "matmul_coopmat"; + kernel_name.reserve(kShaderNameReserve); + add_dtype_suffix(kernel_name, graph->dtype_of(out)); + return VK_KERNEL_FROM_STR(kernel_name); +} + +static GlobalWorkGrid pick_matmul_coopmat_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + (void)shader; + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + const auto out_sizes = graph->sizes_of(out); + uint32_t M = + utils::safe_downcast(out_sizes.at(out_sizes.size() - 2)); + uint32_t N = + utils::safe_downcast(out_sizes.at(out_sizes.size() - 1)); + uint32_t num_tiles_n = utils::div_up(N, kCoopmatTileN); + uint32_t num_tiles_m = utils::div_up(M, kCoopmatTileM); + // Each workgroup processes one WG_TILE_M x WG_TILE_N output tile via + // cooperative-matrix MMAs across its 4 subgroups. We want the dispatch + // to launch exactly num_tiles_n x num_tiles_m workgroups. + // + // The framework computes the group count as + // group_count = div_up(gwg, lwg) + // (see Context.cpp + Command.cpp). With lwg = (kCoopmatInvocations, + // 1, 1), multiplying num_tiles_n by kCoopmatInvocations cancels the + // div, yielding group_count.x = num_tiles_n. + return GlobalWorkGrid( + {num_tiles_n * kCoopmatInvocations, num_tiles_m, 1u}, + kTiledWorkGrid, + LocalWorkGroup(kCoopmatInvocations, 1u, 1u)); +} + +void add_matmul_coopmat_node( + ComputeGraph& graph, + const ValueRef mat1, + const ValueRef mat2, + const ValueRef out) { + VK_CHECK_COND(graph.packed_dim_of(mat1) == WHCN::kWidthDim); + VK_CHECK_COND(graph.packed_dim_of(mat2) == WHCN::kWidthDim); + VK_CHECK_COND(graph.packed_dim_of(out) == WHCN::kWidthDim); + VK_CHECK_COND( + graph.storage_type_of(out) == utils::kBuffer, + "matmul_coopmat requires buffer storage"); + + ValueRef has_bias_ref = graph.add_scalar(false); + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + pick_matmul_coopmat_shader, + pick_matmul_coopmat_gwg, + pick_required_lwg, + // Inputs and Outputs — same binding order as matmul_vec + {{out, vkapi::kWrite}, {{mat1, mat2}, vkapi::kRead}}, + // Shader params buffers — same UBOs as matmul_vec + {graph.sizes_ubo(mat1), graph.sizes_ubo(mat2)}, + // Push Constants + {}, + // Specialization Constants + {}, + // Resize Args + {has_bias_ref}, + // Resizing Logic + resize_matmul_tiled_node)); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.h b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.h new file mode 100644 index 00000000000..4cf825fc838 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +void add_matmul_coopmat_node( + ComputeGraph& graph, + const ValueRef mat1, + const ValueRef mat2, + const ValueRef out); + +} // namespace vkcompute diff --git a/backends/vulkan/test/custom_ops/impl/TestFpaQ4gswLinear.cpp b/backends/vulkan/test/custom_ops/impl/TestFpaQ4gswLinear.cpp index a9e5ad96e95..2da2cd6ceb7 100644 --- a/backends/vulkan/test/custom_ops/impl/TestFpaQ4gswLinear.cpp +++ b/backends/vulkan/test/custom_ops/impl/TestFpaQ4gswLinear.cpp @@ -10,9 +10,9 @@ #include #include -#include #include #include +#include #include namespace vkcompute { diff --git a/backends/vulkan/test/custom_ops/impl/TestMatmulLinear.cpp b/backends/vulkan/test/custom_ops/impl/TestMatmulLinear.cpp index 60eeb2b034c..fc5fb20cab4 100644 --- a/backends/vulkan/test/custom_ops/impl/TestMatmulLinear.cpp +++ b/backends/vulkan/test/custom_ops/impl/TestMatmulLinear.cpp @@ -9,11 +9,12 @@ #include #include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include #include namespace vkcompute { From 8321e1d2d6776204c31b8383e773180233c5c51e Mon Sep 17 00:00:00 2001 From: Stephen Jia Date: Mon, 12 Oct 2026 12:45:48 -0700 Subject: [PATCH 6/7] [ET-VK][test] Move operator benchmark binaries under ops/benchmark Pull Request resolved: https://github.com/pytorch/executorch/pull/23597 ## Ulterior Motive Restructure the Vulkan operator test+benchmark binaries around a single `etvk_benchmark` entry point with registered, per-operator test case sets, so related cases (such as a model's bottleneck shapes) can be grouped and run in isolation. ## Rationale **What**: Move the test+benchmark binaries, their shared utilities, and the test-only ops and shaders from `test/custom_ops/` into `test/ops/benchmark/`, with one subdirectory per operator under `cases/`, and split the monolithic `utils.{h,cpp}` by topic. **Why**: The flat `custom_ops/` directory mixes benchmark binaries, framework code, test-only ops, and correctness-only gtests, and `utils.{h,cpp}` (3.5k lines) mixes value specs, test cases, execution, and reporting. Grouping the binaries by operator gives the upcoming case-set registration a home for each operator's cases and reference code. ## Details - Layout under `test/ops/benchmark/` - `framework/`: test framework utilities, `conv2d_utils`, `cm_utils`, and `test_coopmat_probe`. - `test_ops/impl/` and `test_ops/glsl/`: test-only ops and shaders, formerly `impl/` and `glsl/`. - `cases/`: one directory per operator, `conv2d/` (direct, depthwise, pointwise), `conv1d/`, `q8csw_conv2d/`, `q4gsw_linear/` (including `test_fpa_q4gsw_linear` and `test_coopmat_linear_bench`), `q8csw_linear/`, `mm/`, `sdpa/`, `embedding_q4gsw/`, `choose_qparams_per_row/`, `add/`; `cases/q8ta/` collects the q8ta operators: `conv2d/`, `conv2d_transposed/`, `linear/`, `binary/`, `unary/`, `clone/`, `pixel_shuffle/`, `qdq/`. - `utils.{h,cpp}` split - `config`: global output and debugging options. - `labels.h`: test case naming helpers. - `value_spec`: `ValueSpec`, data generation, half-precision conversion, and `print_valuespec_data`. - `test_case`: `ReferenceKey`, the shader filter defaults, and `TestCase`. - `results`: `BenchmarkResult`, `TestResult`, and table printing. - `runner`: graph setup, `RepeatedGraphExecutor`, and `execute_test_case(s)`. - `weight_utils`: weight canvases and weight sum helpers. - `utils.h` includes all of the above, so binaries are unchanged. - Moving code into new files exposes it to CLANGTIDY, so its findings are fixed: braces around single-statement bodies, `explicit` on the single-argument `BenchmarkResult`, `TestResult`, and `TestCase` constructors, `const&` for the generator, reference, FLOP calculator, and `op_name` parameters, `utils::safe_downcast` for one narrowing, `(void)` for an unused parameter, and `NOLINTNEXTLINE` on enum `switch`es that list only some enumerators (each has a `default`) and on the `using namespace vkcompute` that binaries rely on. `utils.h` no longer includes ``. - Build - `test/ops/benchmark/{BUCK,targets.bzl}` define the binaries, `prototyping_utils`, `operator_implementations`, and `custom_ops_shaderlib`. Binary target names are unchanged; only the package path changes. - The CMake project and `build_and_run.sh` move with the binaries and take source paths from the new subdirectories. - The correctness-only gtests (`utils_test`, `q8ta_conv2d_route_test`, `q8ta_conv2d_stream_plan_test`) stay in `test/custom_ops/`; `utils_test` now depends on `prototyping_utils` in its new package. - Binaries include framework headers by full path (`executorch/backends/vulkan/test/ops/benchmark/framework/utils.h`). - CI references move to the new package: the GPU-host and Eureka Skycastle workflows, and the OSS `vulkan.yml` and `pull.yml` workflows. - Moved files are recorded as renames; the new `BUCK`/`targets.bzl` and the split framework files are recorded as copies. Binary sources, test cases, and test-only ops are otherwise unchanged. ghstack-source-id: 446951727 @exported-using-ghexport Differential Revision: [D123690842](https://our.internmc.facebook.com/intern/diff/D123690842/) --- .github/workflows/pull.yml | 16 +- .github/workflows/vulkan.yml | 18 +- backends/vulkan/test/custom_ops/targets.bzl | 104 +- backends/vulkan/test/custom_ops/utils.cpp | 2482 ----------------- backends/vulkan/test/custom_ops/utils.h | 1094 -------- .../vulkan/test/custom_ops/utils_test.cpp | 2 +- .../op_tests/conv2d_gemm_dynamic_test.cpp | 13 +- backends/vulkan/test/ops/benchmark/BUCK | 9 + .../benchmark}/CMakeLists.txt | 55 +- .../benchmark}/build_and_run.sh | 14 +- .../benchmark/cases/add}/test_add.cpp | 2 +- .../test_choose_qparams_per_row.cpp | 2 +- .../cases/conv1d}/test_conv1d_dw.cpp | 2 +- .../cases/conv1d}/test_conv1d_pw.cpp | 2 +- .../benchmark/cases/conv2d}/test_conv2d.cpp | 4 +- .../cases/conv2d}/test_conv2d_dw.cpp | 4 +- .../cases/conv2d}/test_conv2d_pw.cpp | 2 +- .../embedding_q4gsw}/test_embedding_q4gsw.cpp | 2 +- .../benchmark/cases/mm}/test_mm.cpp | 2 +- .../test_coopmat_linear_bench.cpp | 2 +- .../q4gsw_linear}/test_fpa_q4gsw_linear.cpp | 2 +- .../cases/q4gsw_linear}/test_q4gsw_linear.cpp | 2 +- .../cases/q8csw_conv2d}/test_q8csw_conv2d.cpp | 4 +- .../cases/q8csw_linear}/test_q8csw_linear.cpp | 2 +- .../cases/q8ta/binary}/test_q8ta_binary.cpp | 2 +- .../cases/q8ta/clone}/test_q8ta_clone.cpp | 2 +- .../cases/q8ta/conv2d}/test_q8ta_conv2d.cpp | 4 +- .../q8ta/conv2d}/test_q8ta_conv2d_dw.cpp | 4 +- .../q8ta/conv2d}/test_q8ta_conv2d_pw.cpp | 4 +- .../test_q8ta_conv2d_transposed.cpp | 4 +- .../cases/q8ta/linear}/test_q8ta_linear.cpp | 2 +- .../test_q8ta_pixel_shuffle.cpp | 2 +- .../cases/q8ta/qdq}/test_q8ta_qdq.cpp | 2 +- .../cases/q8ta/unary}/test_q8ta_unary.cpp | 2 +- .../benchmark/cases/sdpa}/test_sdpa.cpp | 2 +- .../benchmark/framework}/cm_utils.cpp | 0 .../benchmark/framework}/cm_utils.h | 0 .../test/ops/benchmark/framework/config.cpp | 55 + .../test/ops/benchmark/framework/config.h | 31 + .../benchmark/framework}/conv2d_utils.cpp | 0 .../benchmark/framework}/conv2d_utils.h | 0 .../test/ops/benchmark/framework/labels.h | 136 + .../test/ops/benchmark/framework/results.cpp | 468 ++++ .../test/ops/benchmark/framework/results.h | 235 ++ .../test/ops/benchmark/framework/runner.cpp | 760 +++++ .../test/ops/benchmark/framework/runner.h | 123 + .../ops/benchmark/framework/test_case.cpp | 79 + .../test/ops/benchmark/framework/test_case.h | 236 ++ .../framework}/test_coopmat_probe.cpp | 0 .../test/ops/benchmark/framework/utils.h | 27 + .../ops/benchmark/framework/value_spec.cpp | 1024 +++++++ .../test/ops/benchmark/framework/value_spec.h | 404 +++ .../ops/benchmark/framework/weight_utils.cpp | 247 ++ .../ops/benchmark/framework/weight_utils.h | 52 + .../vulkan/test/ops/benchmark/targets.bzl | 129 + .../benchmark/test_ops}/glsl/add.yaml | 0 .../benchmark/test_ops}/glsl/add_buffer.glsl | 0 .../benchmark/test_ops}/glsl/add_texture.glsl | 0 .../test_ops}/glsl/float_canvas.glsl | 0 .../glsl/packed_int32_canvas_buffer.glsl | 0 .../glsl/packed_int32_canvas_texture3d.glsl | 0 .../glsl/q4gsw_linear_gemv__w_4x8.glsl | 0 .../glsl/q4gsw_linear_gemv__w_4x8.yaml | 0 .../benchmark/test_ops}/impl/AddPrototype.cpp | 0 .../benchmark/test_ops}/impl/TestConv1dDW.cpp | 0 .../benchmark/test_ops}/impl/TestConv1dPW.cpp | 0 .../benchmark/test_ops}/impl/TestConv2d.cpp | 0 .../benchmark/test_ops}/impl/TestConv2dDw.cpp | 0 .../benchmark/test_ops}/impl/TestConv2dPw.cpp | 0 .../test_ops}/impl/TestFpaQ4gswLinear.cpp | 8 +- .../test_ops}/impl/TestMatmulLinear.cpp | 0 .../test_ops}/impl/TestQ8taBinary.cpp | 0 .../test_ops}/impl/TestQ8taClone.cpp | 0 .../test_ops}/impl/TestQ8taConv2d.cpp | 0 .../impl/TestQ8taConv2dTransposed.cpp | 0 .../test_ops}/impl/TestQ8taLinear.cpp | 0 .../test_ops}/impl/TestQ8taPixelShuffle.cpp | 0 .../impl/TestQ8taQuantizeDequantize.cpp | 0 .../test_ops}/impl/TestQ8taUnary.cpp | 0 .../benchmark/test_ops}/impl/TestSDPA.cpp | 0 80 files changed, 4117 insertions(+), 3768 deletions(-) delete mode 100644 backends/vulkan/test/custom_ops/utils.cpp delete mode 100644 backends/vulkan/test/custom_ops/utils.h create mode 100644 backends/vulkan/test/ops/benchmark/BUCK rename backends/vulkan/test/{custom_ops => ops/benchmark}/CMakeLists.txt (63%) rename backends/vulkan/test/{custom_ops => ops/benchmark}/build_and_run.sh (89%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/add}/test_add.cpp (98%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/choose_qparams_per_row}/test_choose_qparams_per_row.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/conv1d}/test_conv1d_dw.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/conv1d}/test_conv1d_pw.cpp (98%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/conv2d}/test_conv2d.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/conv2d}/test_conv2d_dw.cpp (98%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/conv2d}/test_conv2d_pw.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/embedding_q4gsw}/test_embedding_q4gsw.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/mm}/test_mm.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q4gsw_linear}/test_coopmat_linear_bench.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q4gsw_linear}/test_fpa_q4gsw_linear.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q4gsw_linear}/test_q4gsw_linear.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q8csw_conv2d}/test_q8csw_conv2d.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q8csw_linear}/test_q8csw_linear.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q8ta/binary}/test_q8ta_binary.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q8ta/clone}/test_q8ta_clone.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q8ta/conv2d}/test_q8ta_conv2d.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q8ta/conv2d}/test_q8ta_conv2d_dw.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q8ta/conv2d}/test_q8ta_conv2d_pw.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q8ta/conv2d_transposed}/test_q8ta_conv2d_transposed.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q8ta/linear}/test_q8ta_linear.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q8ta/pixel_shuffle}/test_q8ta_pixel_shuffle.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q8ta/qdq}/test_q8ta_qdq.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/q8ta/unary}/test_q8ta_unary.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/cases/sdpa}/test_sdpa.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/framework}/cm_utils.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/framework}/cm_utils.h (100%) create mode 100644 backends/vulkan/test/ops/benchmark/framework/config.cpp create mode 100644 backends/vulkan/test/ops/benchmark/framework/config.h rename backends/vulkan/test/{custom_ops => ops/benchmark/framework}/conv2d_utils.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/framework}/conv2d_utils.h (100%) create mode 100644 backends/vulkan/test/ops/benchmark/framework/labels.h create mode 100644 backends/vulkan/test/ops/benchmark/framework/results.cpp create mode 100644 backends/vulkan/test/ops/benchmark/framework/results.h create mode 100644 backends/vulkan/test/ops/benchmark/framework/runner.cpp create mode 100644 backends/vulkan/test/ops/benchmark/framework/runner.h create mode 100644 backends/vulkan/test/ops/benchmark/framework/test_case.cpp create mode 100644 backends/vulkan/test/ops/benchmark/framework/test_case.h rename backends/vulkan/test/{custom_ops => ops/benchmark/framework}/test_coopmat_probe.cpp (100%) create mode 100644 backends/vulkan/test/ops/benchmark/framework/utils.h create mode 100644 backends/vulkan/test/ops/benchmark/framework/value_spec.cpp create mode 100644 backends/vulkan/test/ops/benchmark/framework/value_spec.h create mode 100644 backends/vulkan/test/ops/benchmark/framework/weight_utils.cpp create mode 100644 backends/vulkan/test/ops/benchmark/framework/weight_utils.h create mode 100644 backends/vulkan/test/ops/benchmark/targets.bzl rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/glsl/add.yaml (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/glsl/add_buffer.glsl (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/glsl/add_texture.glsl (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/glsl/float_canvas.glsl (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/glsl/packed_int32_canvas_buffer.glsl (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/glsl/packed_int32_canvas_texture3d.glsl (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/glsl/q4gsw_linear_gemv__w_4x8.glsl (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/glsl/q4gsw_linear_gemv__w_4x8.yaml (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/AddPrototype.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestConv1dDW.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestConv1dPW.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestConv2d.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestConv2dDw.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestConv2dPw.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestFpaQ4gswLinear.cpp (99%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestMatmulLinear.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestQ8taBinary.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestQ8taClone.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestQ8taConv2d.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestQ8taConv2dTransposed.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestQ8taLinear.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestQ8taPixelShuffle.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestQ8taQuantizeDequantize.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestQ8taUnary.cpp (100%) rename backends/vulkan/test/{custom_ops => ops/benchmark/test_ops}/impl/TestSDPA.cpp (100%) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 97ca6969318..925d2a6e367 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -1680,14 +1680,14 @@ jobs: .ci/scripts/setup-linux.sh --build-tool "cmake" # Custom operator tests - PYTHON_EXECUTABLE=python bash backends/vulkan/test/custom_ops/build_and_run.sh test_add - ./cmake-out/backends/vulkan/test/custom_ops/test_q8csw_linear - ./cmake-out/backends/vulkan/test/custom_ops/test_q8csw_conv2d - ./cmake-out/backends/vulkan/test/custom_ops/test_q4gsw_linear - ./cmake-out/backends/vulkan/test/custom_ops/test_choose_qparams_per_row - ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_qdq - ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_clone - ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_binary + PYTHON_EXECUTABLE=python bash backends/vulkan/test/ops/benchmark/build_and_run.sh test_add + ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8csw_linear + ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8csw_conv2d + ./cmake-out/backends/vulkan/test/ops/benchmark/test_q4gsw_linear + ./cmake-out/backends/vulkan/test/ops/benchmark/test_choose_qparams_per_row + ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8ta_qdq + ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8ta_clone + ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8ta_binary # "Classic" Operator tests PYTHON_EXECUTABLE=python bash backends/vulkan/test/scripts/test_op.sh --build diff --git a/.github/workflows/vulkan.yml b/.github/workflows/vulkan.yml index 223fadebf23..fd9919ade35 100644 --- a/.github/workflows/vulkan.yml +++ b/.github/workflows/vulkan.yml @@ -100,15 +100,15 @@ jobs: # Operator coverage (mirrors test-vulkan-operators-linux, on real hardware). # The custom-op prototyping binaries are GPU microbenchmarks that rely on # GPU timestamp queries, so they require the hardware selected above. - PYTHON_EXECUTABLE=python bash backends/vulkan/test/custom_ops/build_and_run.sh - ./cmake-out/backends/vulkan/test/custom_ops/test_add - ./cmake-out/backends/vulkan/test/custom_ops/test_q8csw_linear - ./cmake-out/backends/vulkan/test/custom_ops/test_q8csw_conv2d - ./cmake-out/backends/vulkan/test/custom_ops/test_q4gsw_linear - ./cmake-out/backends/vulkan/test/custom_ops/test_choose_qparams_per_row - ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_qdq - ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_clone - ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_binary + PYTHON_EXECUTABLE=python bash backends/vulkan/test/ops/benchmark/build_and_run.sh + ./cmake-out/backends/vulkan/test/ops/benchmark/test_add + ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8csw_linear + ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8csw_conv2d + ./cmake-out/backends/vulkan/test/ops/benchmark/test_q4gsw_linear + ./cmake-out/backends/vulkan/test/ops/benchmark/test_choose_qparams_per_row + ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8ta_qdq + ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8ta_clone + ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8ta_binary PYTHON_EXECUTABLE=python bash backends/vulkan/test/scripts/test_op.sh --build vulkan_sdpa_av_test diff --git a/backends/vulkan/test/custom_ops/targets.bzl b/backends/vulkan/test/custom_ops/targets.bzl index e3631298119..2355e6a4f2a 100644 --- a/backends/vulkan/test/custom_ops/targets.bzl +++ b/backends/vulkan/test/custom_ops/targets.bzl @@ -1,89 +1,14 @@ -load("@fbsource//tools/build_defs:platform_defs.bzl", "ANDROID", "CXX") +load("@fbsource//tools/build_defs:platform_defs.bzl", "CXX") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") load( "@fbsource//xplat/executorch/backends/vulkan:targets.bzl", "get_platforms", - "vulkan_spv_shader_lib", ) -def define_custom_op_test_binary(custom_op_name, extra_deps = [], src_file = None, include_torch = False): - deps_list = [ - ":prototyping_utils", - ":operator_implementations", - ":custom_ops_shaderlib", - "//executorch/backends/vulkan:vulkan_graph_runtime", - ] + ([runtime.external_dep_location("libtorch")] if include_torch else []) + extra_deps - - src_file_str = src_file if src_file else "{}.cpp".format(custom_op_name) - - runtime.cxx_binary( - name = custom_op_name, - srcs = [ - src_file_str, - ], - platforms = get_platforms(), - define_static_target = False, - deps = deps_list, - ) - def define_common_targets(is_fbcode = False): if is_fbcode: return - # Shader library from GLSL files - runtime.filegroup( - name = "custom_ops_shaders", - srcs = native.glob([ - "glsl/**/*", - ]), - visibility = ["PUBLIC"], - ) - - vulkan_spv_shader_lib( - name = "custom_ops_shaderlib", - spv_filegroups = { - ":custom_ops_shaders": "glsl", - }, - is_fbcode = is_fbcode, - ) - - # Prototyping utilities library - runtime.cxx_library( - name = "prototyping_utils", - srcs = [ - "utils.cpp", - "conv2d_utils.cpp", - ], - headers = [ - "utils.h", - "conv2d_utils.h", - ], - exported_headers = [ - "utils.h", - "conv2d_utils.h", - ], - platforms = get_platforms(), - deps = [ - "//executorch/backends/vulkan:vulkan_graph_runtime", - ], - visibility = ["PUBLIC"], - ) - - # Operator implementations library - runtime.cxx_library( - name = "operator_implementations", - srcs = native.glob([ - "impl/*.cpp", - ]), - platforms = get_platforms(), - deps = [ - "//executorch/backends/vulkan:vulkan_graph_runtime", - ":custom_ops_shaderlib", - ], - visibility = ["PUBLIC"], - link_whole = True, - ) - runtime.cxx_test( name = "utils_test", srcs = [ @@ -92,7 +17,7 @@ def define_common_targets(is_fbcode = False): contacts = ["oncall+ai_infra_mobile_platform@xmail.facebook.com"], platforms = [CXX], deps = [ - ":prototyping_utils", + "//executorch/backends/vulkan/test/ops/benchmark:prototyping_utils", "//third-party/googletest:gtest_main", ], ) @@ -116,28 +41,3 @@ def define_common_targets(is_fbcode = False): "//executorch/backends/vulkan:vulkan_graph_runtime", ], ) - - define_custom_op_test_binary("test_add") - define_custom_op_test_binary("test_q8csw_linear") - define_custom_op_test_binary("test_q8csw_conv2d") - define_custom_op_test_binary("test_choose_qparams_per_row") - define_custom_op_test_binary("test_q4gsw_linear") - define_custom_op_test_binary("test_q8ta_qdq") - define_custom_op_test_binary("test_q8ta_clone") - define_custom_op_test_binary("test_q8ta_binary") - define_custom_op_test_binary("test_q8ta_conv2d") - define_custom_op_test_binary("test_q8ta_conv2d_pw") - define_custom_op_test_binary("test_q8ta_conv2d_dw") - define_custom_op_test_binary("test_q8ta_linear") - define_custom_op_test_binary("test_q8ta_conv2d_transposed") - define_custom_op_test_binary("test_q8ta_pixel_shuffle") - define_custom_op_test_binary("test_q8ta_unary") - define_custom_op_test_binary("test_mm") - define_custom_op_test_binary("test_conv2d") - define_custom_op_test_binary("test_conv2d_pw") - define_custom_op_test_binary("test_conv2d_dw") - define_custom_op_test_binary("test_embedding_q4gsw") - define_custom_op_test_binary("test_conv1d_pw") - define_custom_op_test_binary("test_conv1d_dw") - define_custom_op_test_binary("test_fpa_q4gsw_linear") - define_custom_op_test_binary("test_sdpa") diff --git a/backends/vulkan/test/custom_ops/utils.cpp b/backends/vulkan/test/custom_ops/utils.cpp deleted file mode 100644 index 066996de262..00000000000 --- a/backends/vulkan/test/custom_ops/utils.cpp +++ /dev/null @@ -1,2482 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include "utils.h" -#include -#include -#include -#include -#include - -#include - -namespace executorch { -namespace vulkan { -namespace prototyping { - -namespace { - -// Upper bound on the auto-sized chained_dispatches factor. Caps the factor -// when an op is fast enough that target_us / probe_us would otherwise -// exceed this; past this point, stacking more dispatches doesn't materially -// improve measurement precision. -constexpr int kMaxChainedDispatches = 200; - -// Floor for probe_us to avoid division-by-zero / a runaway chained_dispatches -// factor when an op finishes faster than the wall clock can resolve. -constexpr float kMinProbeTimeUs = 1.0f; - -} // namespace - -// Benchmark graphs are immutable after input upload, so the operator nodes can -// be recorded repeatedly. Staging uploads/downloads are encoded once: -// repeating them would redo host-visible copies on every repetition and bias -// per-invocation timings. -RepeatedGraphExecutor::RepeatedGraphExecutor( - ComputeGraph& graph, - int repetitions, - OpNodeRange op_nodes) - : graph_(graph) { - VK_CHECK_COND(repetitions > 0); - VK_CHECK_COND(op_nodes.begin <= op_nodes.end); - VK_CHECK_COND(op_nodes.end <= graph_.execute_nodes().size()); - - api::Context* const context = graph_.context(); - context->flush(); - context->set_cmd(/*reusable=*/true); - context->cmd_reset_querypool(); - - for (size_t i = 0; i < op_nodes.begin; ++i) { - graph_.execute_nodes()[i]->encode(&graph_); - } - for (int i = 0; i < repetitions; ++i) { - for (size_t j = op_nodes.begin; j < op_nodes.end; ++j) { - graph_.execute_nodes()[j]->encode(&graph_); - } - } - for (size_t i = op_nodes.end; i < graph_.execute_nodes().size(); ++i) { - graph_.execute_nodes()[i]->encode(&graph_); - } - - command_ = - std::make_unique(std::move(context->extract_cmd())); -} - -void RepeatedGraphExecutor::execute() { - api::Context* const context = graph_.context(); - command_->end(); - - // Intentionally bypasses Context::submit_cmd_to_gpu(): its submit-count - // bookkeeping and threshold-split path only serve submit_compute_job - // recording, which benchmarks don't use. - vkapi::VulkanFence fence = context->fences().get_fence(); - context->adapter_ptr()->submit_cmd( - context->queue(), - command_->get_submit_handle(/*final_use=*/false), - fence.get_submit_handle()); - fence.wait(); - context->fences().return_fence(fence); -} - -int get_seed() { - static int seed = 42; - return seed++; -} - -int get_seed_or_explicit(int explicit_seed) { - if (explicit_seed >= 0) { - return explicit_seed; - } - return get_seed(); -} - -// Forward declarations for data generation utilities -void generate_random_float_data( - std::vector& data, - float min_val = -1.0f, - float max_val = 1.0f, - int explicit_seed = -1); -void generate_random_int_data( - std::vector& data, - int min_val = -10, - int max_val = 10, - int explicit_seed = -1); -void generate_randint_float_data( - std::vector& data, - int min_val = -10, - int max_val = 10, - int explicit_seed = -1); -void generate_randint_half_data( - std::vector& data, - int min_val = -10, - int max_val = 10, - int explicit_seed = -1); -void generate_random_int8_data( - std::vector& data, - int8_t min_val = -10, - int8_t max_val = 10, - int explicit_seed = -1); -void generate_random_uint8_data( - std::vector& data, - uint8_t min_val = 0, - uint8_t max_val = 255, - int explicit_seed = -1); -void generate_random_2xint4_data( - std::vector& data, - int explicit_seed = -1); -void generate_random_2xint4_data( - std::vector& data, - int explicit_seed = -1); -void generate_random_int4_data( - std::vector& data, - int8_t min_val = -8, - int8_t max_val = 7, - int explicit_seed = -1); -void generate_ones_data(std::vector& data); -void generate_zeros_data(std::vector& data); - -// Convert a float32 value to IEEE 754 half-precision (uint16_t) -uint16_t float_to_half(float value) { - uint32_t float_bits; - std::memcpy(&float_bits, &value, sizeof(float)); - uint32_t sign = (float_bits >> 31) & 0x1; - int32_t exponent = static_cast((float_bits >> 23) & 0xFF) - 127; - uint32_t mantissa = float_bits & 0x7FFFFF; - - uint16_t half_val; - if (exponent > 15) { - half_val = static_cast((sign << 15) | 0x7C00); // Inf - } else if (exponent < -14) { - half_val = static_cast(sign << 15); // Zero / subnormal - } else { - half_val = static_cast( - (sign << 15) | (static_cast(exponent + 15) << 10) | - (mantissa >> 13)); - } - return half_val; -} - -// Convert a IEEE 754 half-precision (uint16_t) value to float32 -float half_to_float(uint16_t half_val) { - uint32_t sign = (half_val >> 15) & 0x1; - uint32_t exponent = (half_val >> 10) & 0x1F; - uint32_t mantissa = half_val & 0x3FF; - - float result; - if (exponent == 0) { - result = std::ldexp(static_cast(mantissa), -24); - } else if (exponent == 31) { - result = mantissa ? std::numeric_limits::quiet_NaN() - : std::numeric_limits::infinity(); - } else { - result = std::ldexp( - 1.0f + static_cast(mantissa) / 1024.0f, exponent - 15); - } - if (sign) { - result = -result; - } - return result; -} - -// Output and latency printing utilities -namespace { -static int print_output_enabled = 0; -static int print_latencies_enabled = 0; -static int gpu_timestamps_enabled = 0; -static int debugging_enabled = 0; -} // namespace - -bool print_output() { - return print_output_enabled > 0; -} - -void set_print_output(bool print_output) { - print_output_enabled = print_output ? 1 : 0; -} - -bool print_latencies() { - return print_latencies_enabled > 0; -} - -void set_print_latencies(bool print_latencies) { - print_latencies_enabled = print_latencies ? 1 : 0; -} - -bool use_gpu_timestamps() { - return gpu_timestamps_enabled > 0; -} - -void set_use_gpu_timestamps(bool use_timestamps) { - gpu_timestamps_enabled = use_timestamps ? 1 : 0; -} - -bool debugging() { - return debugging_enabled > 0; -} - -void set_debugging(bool enable_debugging) { - debugging_enabled = enable_debugging ? 1 : 0; -} - -// ValueSpec implementation -void ValueSpec::ensure_unique_data() const { - if (data_.use_count() != 1) { - data_ = std::make_shared(*data_); - } -} - -void ValueSpec::ensure_unique_reference_data() const { - if (reference_data_.use_count() != 1) { - reference_data_ = std::make_shared(*reference_data_); - } -} - -void ValueSpec::generate_tensor_data(int seed) const { - if (spec_type != SpecType::Tensor) { - return; - } - - ensure_unique_data(); - auto& float_data = data_->float_data; - auto& int32_data = data_->int32_data; - auto& half_data = data_->half_data; - auto& int8_data = data_->int8_data; - auto& uint8_data = data_->uint8_data; - - int64_t num_elements = numel(); - - switch (dtype) { - case vkapi::kFloat: { - float_data.resize(num_elements); - if (data_gen_type == DataGenType::RANDOM) { - generate_random_float_data(float_data, -1.0f, 1.0f, seed); - } else if (data_gen_type == DataGenType::RANDOM_SCALES) { - generate_random_float_data(float_data, 0.005, 0.015, seed); - } else if (data_gen_type == DataGenType::RANDINT) { - generate_randint_float_data(float_data, -10, 10, seed); - } else if (data_gen_type == DataGenType::RANDINT8) { - generate_randint_float_data(float_data, -128, 127, seed); - } else if (data_gen_type == DataGenType::RANDINT4) { - generate_randint_float_data(float_data, -8, 7, seed); - } else if (data_gen_type == DataGenType::ONES) { - generate_ones_data(float_data); - } else if (data_gen_type == DataGenType::ZEROS) { - generate_zeros_data(float_data); - } else { - generate_zeros_data(float_data); - } - break; - } - case vkapi::kHalf: { - half_data.resize(num_elements); - if (data_gen_type == DataGenType::RANDOM) { - // Generate random float data first, then convert to IEEE 754 half. - std::vector temp_data(num_elements); - generate_random_float_data(temp_data, -1.0f, 1.0f, seed); - for (size_t i = 0; i < temp_data.size(); ++i) { - half_data[i] = float_to_half(temp_data[i]); - } - } else if (data_gen_type == DataGenType::RANDOM_SCALES) { - // Generate random scales in float, then convert to proper fp16 - std::vector temp_data(num_elements); - generate_random_float_data(temp_data, 0.005f, 0.015f, seed); - for (size_t i = 0; i < temp_data.size(); ++i) { - half_data[i] = float_to_half(temp_data[i]); - } - } else if (data_gen_type == DataGenType::RANDINT) { - generate_randint_half_data(half_data, -10, 10, seed); - } else if (data_gen_type == DataGenType::RANDINT8) { - generate_randint_half_data(half_data, -128, 127, seed); - } else if (data_gen_type == DataGenType::RANDINT4) { - generate_randint_half_data(half_data, -8, 7, seed); - } else if (data_gen_type == DataGenType::ONES) { - std::fill(half_data.begin(), half_data.end(), float_to_half(1.0f)); - } else if (data_gen_type == DataGenType::ZEROS) { - std::fill( - half_data.begin(), - half_data.end(), - static_cast(0)); // 0.0 in half - } else { - std::fill( - half_data.begin(), - half_data.end(), - static_cast(0)); // 0.0 in half - } - break; - } - case vkapi::kInt: { - int32_data.resize(num_elements); - if (data_gen_type == DataGenType::RANDOM) { - generate_random_int_data(int32_data, -10, 10, seed); - } else if (data_gen_type == DataGenType::RANDINT) { - generate_random_int_data( - int32_data, - -10, - 10, - seed); // For int type, RANDINT is same as RANDOM - } else if (data_gen_type == DataGenType::RANDINT8) { - generate_random_int_data(int32_data, -128, 127, seed); - } else if (data_gen_type == DataGenType::RANDINT4) { - generate_random_int_data(int32_data, -8, 7, seed); - } else if (data_gen_type == DataGenType::ONES) { - std::fill(int32_data.begin(), int32_data.end(), 1); - } else if (data_gen_type == DataGenType::ZEROS) { - std::fill(int32_data.begin(), int32_data.end(), 0); - } else { - std::fill(int32_data.begin(), int32_data.end(), 0); - } - break; - } - case vkapi::kChar: { - int8_data.resize(num_elements); - if (data_gen_type == DataGenType::RANDOM) { - generate_random_int8_data(int8_data, -10, 10, seed); - } else if (data_gen_type == DataGenType::RANDINT) { - generate_random_int8_data(int8_data, -10, 10, seed); - } else if (data_gen_type == DataGenType::RANDINT8) { - generate_random_int8_data(int8_data, -128, 127, seed); - } else if (data_gen_type == DataGenType::RANDINT4) { - generate_random_2xint4_data(int8_data, seed); - } else if (data_gen_type == DataGenType::ONES) { - std::fill(int8_data.begin(), int8_data.end(), 1); - } else if (data_gen_type == DataGenType::ONES_INT4) { - int8_t packed_data = (1 << 4) | 1; - std::fill(int8_data.begin(), int8_data.end(), packed_data); - } else if (data_gen_type == DataGenType::ZEROS) { - std::fill(int8_data.begin(), int8_data.end(), 0); - } else { - std::fill(int8_data.begin(), int8_data.end(), 0); - } - break; - } - case vkapi::kByte: { - uint8_data.resize(num_elements); - if (data_gen_type == DataGenType::RANDOM) { - generate_random_uint8_data(uint8_data, 0, 255, seed); - } else if (data_gen_type == DataGenType::RANDINT) { - generate_random_uint8_data(uint8_data, 0, 255, seed); - } else if (data_gen_type == DataGenType::RANDINT8) { - generate_random_uint8_data(uint8_data, 0, 255, seed); - } else if (data_gen_type == DataGenType::RANDINT4) { - generate_random_2xint4_data(uint8_data, seed); - } else if (data_gen_type == DataGenType::ONES) { - std::fill(uint8_data.begin(), uint8_data.end(), 1); - } else if (data_gen_type == DataGenType::ONES_INT4) { - uint8_t packed_data = (9 << 4) | 9; - std::fill(uint8_data.begin(), uint8_data.end(), packed_data); - } else if (data_gen_type == DataGenType::ZEROS) { - std::fill(uint8_data.begin(), uint8_data.end(), 0); - } else { - std::fill(uint8_data.begin(), uint8_data.end(), 0); - } - break; - } - default: - // Default to float - float_data.resize(num_elements); - if (data_gen_type == DataGenType::RANDOM) { - generate_random_float_data(float_data, -1.0f, 1.0f, seed); - } else if (data_gen_type == DataGenType::RANDINT) { - generate_randint_float_data(float_data, -10, 10, seed); - } else if (data_gen_type == DataGenType::ONES) { - generate_ones_data(float_data); - } else if (data_gen_type == DataGenType::ZEROS) { - generate_zeros_data(float_data); - } else { - generate_zeros_data(float_data); - } - break; - } -} - -int64_t ValueSpec::numel() const { - if (spec_type == SpecType::Int || spec_type == SpecType::Float || - spec_type == SpecType::Bool) { - return 1; - } else if (spec_type == SpecType::IntList) { - return sizes.empty() ? 0 : sizes[0]; - } else { // Tensor - int64_t total = 1; - for (int64_t size : sizes) { - total *= size; - } - return total; - } -} - -size_t ValueSpec::nbytes() const { - size_t element_size = 0; - switch (dtype) { - case vkapi::kFloat: - element_size = sizeof(float); - break; - case vkapi::kHalf: - element_size = sizeof(uint16_t); - break; - case vkapi::kInt: - element_size = sizeof(int32_t); - break; - case vkapi::kChar: - element_size = sizeof(int8_t); - break; - case vkapi::kByte: - element_size = sizeof(uint8_t); - break; - default: - element_size = sizeof(float); // Default fallback - break; - } - return numel() * element_size; -} - -std::string ValueSpec::to_string() const { - std::string result = "ValueSpec("; - - switch (spec_type) { - case SpecType::Tensor: - result += "type=Tensor, sizes=["; - break; - case SpecType::IntList: - result += "type=IntList, count="; - result += std::to_string(sizes.empty() ? 0 : sizes[0]); - result += ", data_gen="; - result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; - result += ")"; - return result; - case SpecType::Int: - result += "type=Int, value="; - result += std::to_string(get_int_value()); - result += ", data_gen="; - result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; - result += ")"; - return result; - case SpecType::Float: - result += "type=Float, value="; - result += std::to_string(get_float_value()); - result += ", data_gen="; - result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; - result += ")"; - return result; - case SpecType::Bool: - result += "type=Bool, value="; - result += get_bool_value() ? "true" : "false"; - result += ", data_gen="; - result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; - result += ")"; - return result; - case SpecType::String: - result += "type=String, value=\""; - result += get_string_value(); - result += "\")"; - return result; - } - - for (size_t i = 0; i < sizes.size(); ++i) { - result += std::to_string(sizes[i]); - if (i < sizes.size() - 1) - result += ", "; - } - result += "]"; - - if (spec_type == SpecType::Tensor) { - result += ", dtype="; - switch (dtype) { - case vkapi::kFloat: - result += "float"; - break; - case vkapi::kHalf: - result += "half"; - break; - case vkapi::kInt: - result += "int32"; - break; - case vkapi::kChar: - result += "int8"; - break; - case vkapi::kByte: - result += "uint8"; - break; - default: - result += "unknown"; - break; - } - - result += ", memory_layout="; - switch (memory_layout) { - case utils::kWidthPacked: - result += "WidthPacked"; - break; - case utils::kHeightPacked: - result += "HeightPacked"; - break; - case utils::kChannelsPacked: - result += "ChannelsPacked"; - break; - default: - result += "unknown"; - break; - } - - result += ", storage_type="; - switch (storage_type) { - case utils::kTexture3D: - result += "Texture3D"; - break; - case utils::kBuffer: - result += "Buffer"; - break; - default: - result += "unknown"; - break; - } - } - - result += ", data_gen="; - switch (data_gen_type) { - case DataGenType::FIXED: - result += "FIXED"; - break; - case DataGenType::RANDOM: - result += "RANDOM"; - break; - case DataGenType::RANDINT: - result += "RANDINT"; - break; - case DataGenType::RANDINT8: - result += "RANDINT8"; - break; - case DataGenType::RANDINT4: - result += "RANDINT4"; - break; - case DataGenType::ONES: - result += "ONES"; - break; - case DataGenType::ZEROS: - result += "ZEROS"; - break; - default: - result += "unknown"; - break; - } - result += ")"; - return result; -} - -// Additional ValueSpec methods -void ValueSpec::resize_data(size_t new_size) { - // Generate first so a deferred tensor keeps its data-gen pattern (resized, - // not pinned to zeros by the data_generated_ flag set below). - ensure_data_generated(); - ensure_unique_data(); - switch (dtype) { - case vkapi::kFloat: - data_->float_data.resize(new_size); - break; - case vkapi::kHalf: - data_->half_data.resize(new_size); - break; - case vkapi::kInt: - data_->int32_data.resize(new_size); - break; - case vkapi::kChar: - data_->int8_data.resize(new_size); - break; - case vkapi::kByte: - data_->uint8_data.resize(new_size); - break; - default: - data_->float_data.resize(new_size); - break; - } - data_generated_ = true; -} - -void* ValueSpec::get_mutable_data_ptr() { - ensure_data_generated(); - ensure_unique_data(); - switch (dtype) { - case vkapi::kFloat: - return data_->float_data.data(); - case vkapi::kHalf: - return data_->half_data.data(); - case vkapi::kInt: - return data_->int32_data.data(); - case vkapi::kChar: - return data_->int8_data.data(); - case vkapi::kByte: - return data_->uint8_data.data(); - default: - return data_->float_data.data(); - } -} - -float ValueSpec::get_element(size_t index) const { - ensure_data_generated(); - if (index >= static_cast(numel())) { - return 0.0f; - } - - switch (dtype) { - case vkapi::kFloat: - return index < data_->float_data.size() ? data_->float_data[index] : 0.0f; - case vkapi::kHalf: - return index < data_->half_data.size() - ? half_to_float(data_->half_data[index]) - : 0.0f; - case vkapi::kInt: - return index < data_->int32_data.size() - ? static_cast(data_->int32_data[index]) - : 0.0f; - case vkapi::kChar: - return index < data_->int8_data.size() - ? static_cast(data_->int8_data[index]) - : 0.0f; - case vkapi::kByte: - return index < data_->uint8_data.size() - ? static_cast(data_->uint8_data[index]) - : 0.0f; - default: - return 0.0f; - } -} - -const void* ValueSpec::get_data_ptr() const { - ensure_data_generated(); - switch (dtype) { - case vkapi::kFloat: - return data_->float_data.data(); - case vkapi::kHalf: - return data_->half_data.data(); - case vkapi::kInt: - return data_->int32_data.data(); - case vkapi::kChar: - return data_->int8_data.data(); - case vkapi::kByte: - return data_->uint8_data.data(); - default: - throw std::runtime_error("Unsupported data type for get_data_ptr"); - } -} - -void generate_random_float_data( - std::vector& data, - float min_val, - float max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_real_distribution dis(min_val, max_val); - for (auto& val : data) { - val = dis(gen); - } -} - -void generate_random_int_data( - std::vector& data, - int min_val, - int max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(min_val, max_val); - for (auto& val : data) { - val = dis(gen); - } -} - -void generate_randint_float_data( - std::vector& data, - int min_val, - int max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(min_val, max_val); - for (auto& val : data) { - val = static_cast(dis(gen)); - } -} - -void generate_randint_half_data( - std::vector& data, - int min_val, - int max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(min_val, max_val); - for (auto& val : data) { - val = float_to_half(static_cast(dis(gen))); - } -} - -void generate_ones_data(std::vector& data) { - std::fill(data.begin(), data.end(), 1.0f); -} - -void generate_random_int8_data( - std::vector& data, - int8_t min_val, - int8_t max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(min_val, max_val); - for (auto& val : data) { - val = static_cast(dis(gen)); - } -} - -void generate_random_uint8_data( - std::vector& data, - uint8_t min_val, - uint8_t max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(min_val, max_val); - for (auto& val : data) { - val = static_cast(dis(gen)); - } -} - -void generate_random_int4_data( - std::vector& data, - int8_t min_val, - int8_t max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(min_val, max_val); - for (auto& val : data) { - val = static_cast(dis(gen)); - } -} - -void generate_random_2xint4_data(std::vector& data, int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(-8, 7); // Signed 4-bit range - for (auto& val : data) { - // Generate two separate 4-bit values - int8_t lower_4bits = static_cast(dis(gen)) & 0x0F; - int8_t upper_4bits = static_cast(dis(gen)) & 0x0F; - // Pack them into a single 8-bit value - val = (upper_4bits << 4) | lower_4bits; - } -} - -void generate_random_2xint4_data( - std::vector& data, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(0, 15); // Unsigned 4-bit range - for (auto& val : data) { - // Generate two separate 4-bit values - uint8_t lower_4bits = static_cast(dis(gen)) & 0x0F; - uint8_t upper_4bits = static_cast(dis(gen)) & 0x0F; - // Pack them into a single 8-bit value - val = (upper_4bits << 4) | lower_4bits; - } -} - -void generate_zeros_data(std::vector& data) { - std::fill(data.begin(), data.end(), 0.0f); -} - -// Correctness checking against reference data -bool ValueSpec::validate_against_reference( - float abs_tolerance, - float rel_tolerance) const { - // Only validate float and half tensors. For half tensors, convert the - // computed half data to float for comparison against the fp32 reference. - if (!is_tensor() || (dtype != vkapi::kFloat && dtype != vkapi::kHalf)) { - return true; // Skip validation for non-float/half or non-tensor types - } - - // For kHalf, materialize the GPU output as float so the same tolerance - // machinery can compare against the (always-float) reference data. - std::vector half_as_float; - if (dtype == vkapi::kHalf) { - const auto& half_bits = get_half_data(); - half_as_float.resize(half_bits.size()); - for (size_t i = 0; i < half_bits.size(); ++i) { - half_as_float[i] = half_to_float(half_bits[i]); - } - } - // Materialize computed data as float32 for comparison. The dtype is - // guaranteed to be float or half by the early-out above. - const std::vector& computed_data = - (dtype == vkapi::kHalf) ? half_as_float : get_float_data(); - const auto& reference_data = get_ref_float_data(); - - // Skip validation if no reference data is available - if (reference_data.empty()) { - return true; - } - - // Check if sizes match - if (computed_data.size() != reference_data.size()) { - if (debugging()) { - std::cout << "Size mismatch: computed=" << computed_data.size() - << ", reference=" << reference_data.size() << std::endl; - } - return false; - } - - // Element-wise comparison with both absolute and relative tolerance - size_t num_mismatched = 0; - size_t first_mismatch = 0; - for (size_t i = 0; i < computed_data.size(); ++i) { - float diff = std::abs(computed_data[i] - reference_data[i]); - float abs_ref = std::abs(reference_data[i]); - - // Check if either absolute or relative tolerance condition is satisfied - bool abs_tolerance_ok = diff <= abs_tolerance; - bool rel_tolerance_ok = diff <= rel_tolerance * abs_ref; - - if (!abs_tolerance_ok && !rel_tolerance_ok) { - if (num_mismatched == 0) { - first_mismatch = i; - std::cout << "Mismatch at element " << i - << ": computed=" << computed_data[i] - << ", reference=" << reference_data[i] << ", diff=" << diff - << ", abs_tolerance=" << abs_tolerance - << ", rel_tolerance=" << rel_tolerance - << ", rel_threshold=" << (rel_tolerance * abs_ref) - << std::endl; - } - num_mismatched++; - } - } - if (num_mismatched > 0) { - std::cout << " total mismatched: " << num_mismatched << " / " - << computed_data.size() << " (first at " << first_mismatch << ")" - << std::endl; - // For 2D outputs, print a per-16x16-tile mismatch-count map to expose - // the spatial structure of the failure (e.g. zeroed MMA subtiles). - if (sizes.size() == 2) { - const int64_t Mr = sizes[0]; - const int64_t Nc = sizes[1]; - std::cout << " 16x16-tile mismatch counts (rows=M/16, cols=N/16):" - << std::endl; - for (int64_t ti = 0; ti < (Mr + 15) / 16; ++ti) { - std::cout << " "; - for (int64_t tj = 0; tj < (Nc + 15) / 16; ++tj) { - int count = 0; - for (int64_t r = ti * 16; r < std::min(Mr, (ti + 1) * 16); ++r) { - for (int64_t c = tj * 16; c < std::min(Nc, (tj + 1) * 16); ++c) { - float diff = std::abs( - computed_data[r * Nc + c] - reference_data[r * Nc + c]); - float abs_ref = std::abs(reference_data[r * Nc + c]); - if (diff > abs_tolerance && diff > rel_tolerance * abs_ref) { - count++; - } - } - } - std::cout << std::setw(4) << count; - } - std::cout << std::endl; - } - } - return false; - } - - if (debugging()) { - std::cout << "Correctness validation PASSED" << std::endl; - } - return true; -} - -// Ensure data is generated for this ValueSpec -void ValueSpec::ensure_data_generated(int seed) const { - if (data_generated_) { - return; - } - generate_tensor_data(seed); - data_generated_ = true; -} - -void ValueSpec::share_data_from(const ValueSpec& other) { - if (!is_tensor() || !other.is_tensor()) { - return; - } - // Materialize the source first: sharing an ungenerated payload would let a - // later access materialize each spec independently under different seeds. - other.ensure_data_generated(); - data_ = other.data_; - data_generated_ = true; -} - -void ValueSpec::share_reference_from(const ValueSpec& other) { - if (!is_tensor() || !other.is_tensor()) { - return; - } - reference_data_ = other.reference_data_; -} - -// ReferenceKey implementation -ReferenceKey ReferenceKey::from_test_case(const TestCase& tc) { - std::ostringstream oss; - - // Serialize inputs that affect reference computation - // Skip: storage_type, memory_layout, string values (like impl_selector) - for (size_t i = 0; i < tc.inputs().size(); ++i) { - const ValueSpec& input = tc.inputs()[i]; - oss << "i" << i << ":"; - - if (input.is_tensor()) { - // For tensors: sizes, dtype, data_gen_type, is_constant - oss << "T["; - for (size_t j = 0; j < input.sizes.size(); ++j) { - if (j > 0) - oss << ","; - oss << input.sizes[j]; - } - oss << "]d" << static_cast(input.dtype); - oss << "g" << static_cast(input.data_gen_type); - oss << "c" << (input.is_constant() ? 1 : 0); - oss << "n" << (input.is_none() ? 1 : 0); - } else if (input.is_int()) { - oss << "I" << input.get_int_value(); - } else if (input.is_float()) { - oss << "F" << input.get_float_value(); - } else if (input.is_bool()) { - oss << "B" << (input.get_bool_value() ? 1 : 0); - } else if (input.is_int_list()) { - oss << "L["; - const auto& list = input.get_int_list(); - for (size_t j = 0; j < list.size(); ++j) { - if (j > 0) - oss << ","; - oss << list[j]; - } - oss << "]"; - } - // Skip string inputs (like impl_selector) as they don't affect reference - oss << ";"; - } - - // Also include output shapes for completeness - for (size_t i = 0; i < tc.outputs().size(); ++i) { - const ValueSpec& output = tc.outputs()[i]; - oss << "o" << i << ":["; - for (size_t j = 0; j < output.sizes.size(); ++j) { - if (j > 0) - oss << ","; - oss << output.sizes[j]; - } - oss << "]d" << static_cast(output.dtype) << ";"; - } - - ReferenceKey key; - key.key_string = oss.str(); - return key; -} - -// Helper function to collect GPU timing from querypool -float collect_gpu_timing_us( - ComputeGraph& graph, - const std::vector& shader_filter) { - graph.context()->querypool().extract_results(); - const auto results = graph.context()->querypool().get_shader_timestamp_data(); - if (!results.empty()) { - // Sum durations of all shaders that don't match any pattern in - // shader_filter - float total_duration_us = 0.0f; - for (const auto& shader_result : results) { - bool filtered = false; - // Check if this shader matches any filter pattern - for (const auto& filter_pattern : shader_filter) { - if (shader_result.kernel_name.find(filter_pattern) != - std::string::npos) { - filtered = true; - break; - } - } - - if (!filtered) { - // Calculate duration from start and end times, convert from ns to μs - uint64_t duration_ns = - shader_result.end_time_ns - shader_result.start_time_ns; - total_duration_us += static_cast(duration_ns) / 1000.0f; - } - } - return total_duration_us; - } - return 0.0f; -} - -// Helper function to collect per-shader GPU timing from querypool -// Returns a map of shader_name -> timing_us for non-filtered shaders -std::unordered_map collect_per_shader_timing_us( - ComputeGraph& graph, - const std::vector& shader_filter) { - std::unordered_map shader_timings; - - graph.context()->querypool().extract_results(); - const auto results = graph.context()->querypool().get_shader_timestamp_data(); - for (const auto& shader_result : results) { - bool filtered = false; - // Check if this shader matches any filter pattern - for (const auto& filter_pattern : shader_filter) { - if (shader_result.kernel_name.find(filter_pattern) != std::string::npos) { - filtered = true; - break; - } - } - - if (!filtered) { - // Calculate duration from start and end times, convert from ns to μs - uint64_t duration_ns = - shader_result.end_time_ns - shader_result.start_time_ns; - float duration_us = static_cast(duration_ns) / 1000.0f; - // Accumulate timing for shaders with the same name - shader_timings[shader_result.kernel_name] += duration_us; - } - } - return shader_timings; -} - -// BenchmarkResult implementation -void BenchmarkResult::add_iter_timing(float time_us) { - iter_timings.push_back(time_us); -} - -void BenchmarkResult::add_shader_timing( - const std::string& shader_name, - float time_us, - const uint32_t gwg[3], - const uint32_t lwg[3]) { - // Find existing shader timing or create new one - for (auto& st : shader_timings_) { - if (st.shader_name == shader_name) { - st.iter_timings_us.push_back(time_us); - // Work group sizes should be consistent across iterations - return; - } - } - // Not found, create new entry - ShaderTiming new_timing; - new_timing.shader_name = shader_name; - new_timing.iter_timings_us.push_back(time_us); - new_timing.gwg[0] = gwg[0]; - new_timing.gwg[1] = gwg[1]; - new_timing.gwg[2] = gwg[2]; - new_timing.lwg[0] = lwg[0]; - new_timing.lwg[1] = lwg[1]; - new_timing.lwg[2] = lwg[2]; - shader_timings_.push_back(std::move(new_timing)); -} - -float BenchmarkResult::get_avg_time_us() const { - if (iter_timings.empty()) { - return 0.0f; - } - - float sum = 0.0f; - for (float timing : iter_timings) { - sum += timing; - } - return sum / iter_timings.size(); -} - -float BenchmarkResult::get_min_time_us() const { - if (iter_timings.empty()) { - return 0.0f; - } - - return *std::min_element(iter_timings.begin(), iter_timings.end()); -} - -float BenchmarkResult::get_max_time_us() const { - if (iter_timings.empty()) { - return 0.0f; - } - - return *std::max_element(iter_timings.begin(), iter_timings.end()); -} - -float BenchmarkResult::get_std_dev_us() const { - if (iter_timings.size() <= 1) { - return 0.0f; - } - - float mean = get_avg_time_us(); - float sum_sq_diff = 0.0f; - - for (float timing : iter_timings) { - float diff = timing - mean; - sum_sq_diff += diff * diff; - } - - return std::sqrt(sum_sq_diff / (iter_timings.size() - 1)); -} - -void BenchmarkResult::print_summary( - int case_number, - const std::string& size_info, - float total_gflops) const { - static constexpr int OPERATOR_NAME_WIDTH = 50; - static constexpr int GLOBAL_WG_WIDTH = 16; - static constexpr int LOCAL_WG_WIDTH = 12; - static constexpr int KERNEL_NAME_WIDTH = 80; - static constexpr int SIZE_INFO_WIDTH = 20; - static constexpr int TIMING_WIDTH = 16; - static constexpr int GFLOPS_WIDTH = 14; - static constexpr int CORRECTNESS_WIDTH = 8; - - // Helper to truncate shader names longer than 46 chars to 44 chars + ".." - auto truncate_shader_name = [](const std::string& name) -> std::string { - if (name.length() > 46) { - return name.substr(0, 44) + ".."; - } - return name; - }; - - // Helper to format work group size as (x,y,z) - auto format_wg_size = [](const uint32_t wg[3]) -> std::string { - return "(" + std::to_string(wg[0]) + "," + std::to_string(wg[1]) + "," + - std::to_string(wg[2]) + ")"; - }; - - std::string correctness_str; - switch (correctness_status_) { - case CorrectnessStatus::SKIPPED: - correctness_str = "SKIPPED"; - break; - case CorrectnessStatus::PASSED: - correctness_str = "PASSED"; - break; - case CorrectnessStatus::FAILED: - correctness_str = "FAILED"; - break; - } - - // If we have per-shader timing data, print one line per shader plus overall - if (!shader_timings_.empty()) { - // If only one shader, print a single combined row - if (shader_timings_.size() == 1) { - const auto& st = shader_timings_[0]; - std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) - << truncate_shader_name(st.shader_name) << " " << std::left - << std::setw(GLOBAL_WG_WIDTH) << format_wg_size(st.gwg) - << std::left << std::setw(LOCAL_WG_WIDTH) - << format_wg_size(st.lwg) << std::left - << std::setw(KERNEL_NAME_WIDTH) << get_kernel_name() - << std::right << " " << std::setw(SIZE_INFO_WIDTH) << size_info - << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) - << get_avg_time_us() << " μs " << std::setw(GFLOPS_WIDTH) - << std::fixed << std::setprecision(3) << total_gflops - << " GFLOP/s " << std::setw(CORRECTNESS_WIDTH) - << correctness_str << std::endl; - } else { - // Multiple shaders: print individual shader lines (without GFLOP/s) - for (size_t i = 0; i < shader_timings_.size(); ++i) { - const auto& st = shader_timings_[i]; - float shader_avg_time = st.get_avg_time_us(); - - // Shader lines don't show test case info - std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) - << truncate_shader_name(st.shader_name) << " " << std::left - << std::setw(GLOBAL_WG_WIDTH) << format_wg_size(st.gwg) - << std::left << std::setw(LOCAL_WG_WIDTH) - << format_wg_size(st.lwg) << std::left - << std::setw(KERNEL_NAME_WIDTH) << "" << std::right << " " - << std::setw(SIZE_INFO_WIDTH) << "" << std::setw(TIMING_WIDTH) - << std::fixed << std::setprecision(3) << shader_avg_time - << " μs " << std::setw(GFLOPS_WIDTH) << "" << " " - << std::setw(CORRECTNESS_WIDTH) << "" << std::endl; - } - - // Print overall row with operator name, test case info, total time, and - // GFLOP/s - std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) - << get_operator_name() << " " << std::left - << std::setw(GLOBAL_WG_WIDTH) << "" << std::left - << std::setw(LOCAL_WG_WIDTH) << "" << std::left - << std::setw(KERNEL_NAME_WIDTH) << get_kernel_name() - << std::right << " " << std::setw(SIZE_INFO_WIDTH) << size_info - << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) - << get_avg_time_us() << " μs " << std::setw(GFLOPS_WIDTH) - << std::fixed << std::setprecision(3) << total_gflops - << " GFLOP/s " << std::setw(CORRECTNESS_WIDTH) - << correctness_str << std::endl; - } - - // Print separator line between test cases - } else { - // No per-shader timing data, use the original format - std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) - << get_operator_name() << " " << std::left - << std::setw(GLOBAL_WG_WIDTH) << "" << std::left - << std::setw(LOCAL_WG_WIDTH) << "" << std::left - << std::setw(KERNEL_NAME_WIDTH) << get_kernel_name() << std::right - << " " << std::setw(SIZE_INFO_WIDTH) << size_info - << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) - << get_avg_time_us() << " μs " << std::setw(GFLOPS_WIDTH) - << std::fixed << std::setprecision(3) << total_gflops - << " GFLOP/s " << std::setw(CORRECTNESS_WIDTH) << correctness_str - << std::endl; - } -} - -// TestResult implementation -void TestResult::add_result(const BenchmarkResult& result) { - results_.push_back(result); -} - -void TestResult::add_result(BenchmarkResult&& result) { - results_.push_back(std::move(result)); -} - -void TestResult::print_summary() const { - static constexpr int CASE_WIDTH = 100; - static constexpr int KERNEL_NAME_WIDTH = 20; - static constexpr int TIMING_WIDTH = 12; - static constexpr int PASS_WIDTH = 8; - - if (results_.empty()) { - std::cout << "No results to display" << std::endl; - return; - } - - std::cout << "\n=== " << operation_name_ - << " Performance Summary ===" << std::endl; - print_separator(); - - std::cout << std::left << std::setw(CASE_WIDTH) << "Case" << std::left - << std::setw(KERNEL_NAME_WIDTH) << "Kernel Name" << std::left - << std::setw(TIMING_WIDTH) << "Avg (μs)" << std::left - << std::setw(TIMING_WIDTH) << "Min (μs)" << std::left - << std::setw(TIMING_WIDTH) << "Max (μs)" << std::left - << std::setw(TIMING_WIDTH) << "Std Dev" << std::left - << std::setw(PASS_WIDTH) << "Pass" << std::endl; - print_separator(); - - for (size_t i = 0; i < results_.size(); ++i) { - const auto& result = results_[i]; - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - std::cout << std::left << std::setw(CASE_WIDTH) << i + 1 << std::left - << std::setw(KERNEL_NAME_WIDTH) - << result.get_kernel_name().substr(0, KERNEL_NAME_WIDTH - 1) - << std::left << std::setw(TIMING_WIDTH) << std::fixed - << std::setprecision(3) << result.get_avg_time_us() << std::left - << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) - << result.get_min_time_us() << std::left - << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) - << result.get_max_time_us() << std::left - << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) - << result.get_std_dev_us() << std::left << std::setw(PASS_WIDTH) - << (vulkan_execute_succeeded ? "✓" : "✗") << std::endl; - } - - print_separator(); - std::cout << "Total cases: " << results_.size() - << ", Passed: " << get_passed_count() - << ", Failed: " << get_failed_count() << std::endl; - std::cout << "Overall GFLOP/s: " << std::fixed << std::setprecision(3) - << gflops_ << std::endl; - std::cout << "Overall correctness: " - << (correctness_passed_ ? "PASSED" : "FAILED") << std::endl; -} - -void TestResult::print_detailed_results() const { - if (results_.empty()) { - std::cout << "No results to display" << std::endl; - return; - } - - std::cout << "\n=== " << operation_name_ - << " Detailed Results ===" << std::endl; - - for (size_t i = 0; i < results_.size(); ++i) { - const auto& result = results_[i]; - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - std::cout << "\nCase " << i + 1 << ": " << result.get_kernel_name() - << std::endl; - std::cout << " Iterations: " << result.get_num_iterations() << std::endl; - std::cout << " Average: " << std::fixed << std::setprecision(3) - << result.get_avg_time_us() << " μs" << std::endl; - std::cout << " Min: " << std::fixed << std::setprecision(3) - << result.get_min_time_us() << " μs" << std::endl; - std::cout << " Max: " << std::fixed << std::setprecision(3) - << result.get_max_time_us() << " μs" << std::endl; - std::cout << " Std Dev: " << std::fixed << std::setprecision(3) - << result.get_std_dev_us() << " μs" << std::endl; - std::cout << " Correctness: " - << (vulkan_execute_succeeded ? "PASSED" : "FAILED") << std::endl; - - if (result.get_num_iterations() > 0) { - std::cout << " Individual timings (μs): "; - const auto& timings = result.get_iter_timings(); - for (size_t j = 0; j < std::min(size_t(10), timings.size()); ++j) { - std::cout << std::fixed << std::setprecision(1) << timings[j]; - if (j < std::min(size_t(10), timings.size()) - 1) - std::cout << ", "; - } - if (timings.size() > 10) { - std::cout << " ... (" << (timings.size() - 10) << " more)"; - } - std::cout << std::endl; - } - } - - std::cout << "\nOverall Results:" << std::endl; - std::cout << " Total GFLOP/s: " << std::fixed << std::setprecision(3) - << gflops_ << std::endl; - std::cout << " Overall correctness: " - << (correctness_passed_ ? "PASSED" : "FAILED") << std::endl; -} - -void TestResult::print_statistics() const { - if (results_.empty()) { - std::cout << "No results to display statistics for" << std::endl; - return; - } - - std::cout << "\n=== " << operation_name_ << " Statistics ===" << std::endl; - std::cout << "Total test cases: " << results_.size() << std::endl; - std::cout << "Passed: " << get_passed_count() << std::endl; - std::cout << "Failed: " << get_failed_count() << std::endl; - std::cout << "Success rate: " << std::fixed << std::setprecision(1) - << (100.0f * get_passed_count() / results_.size()) << "%" - << std::endl; - - if (get_passed_count() > 0) { - std::cout << "Total average time: " << std::fixed << std::setprecision(3) - << get_total_avg_time_us() << " μs" << std::endl; - std::cout << "Total GFLOP/s: " << std::fixed << std::setprecision(3) - << get_total_gflops() << std::endl; - - const auto* fastest = get_fastest_result(); - const auto* slowest = get_slowest_result(); - const auto* highest_gflops = get_highest_gflops_result(); - - if (fastest) { - std::cout << "Fastest case: " << fastest->get_kernel_name() << " (" - << std::fixed << std::setprecision(3) - << fastest->get_avg_time_us() << " μs)" << std::endl; - } - - if (slowest) { - std::cout << "Slowest case: " << slowest->get_kernel_name() << " (" - << std::fixed << std::setprecision(3) - << slowest->get_avg_time_us() << " μs)" << std::endl; - } - - if (highest_gflops) { - std::cout << "Best performing case: " << highest_gflops->get_kernel_name() - << " (" << std::fixed << std::setprecision(3) - << highest_gflops->get_avg_time_us() << " μs)" << std::endl; - } - } -} - -void TestResult::print_brief_summary() const { - print_separator(); - std::cout << "Summary Statistics:" << std::endl; - - if (get_passed_count() > 0) { - std::cout << "Average execution time: " << std::fixed - << std::setprecision(3) << get_total_avg_time_us() << " μs" - << std::endl; - std::cout << "Total throughput: " << std::fixed << std::setprecision(3) - << get_gflops() << " GFLOP/s" << std::endl; - std::cout << "Successful test cases: " << get_passed_count() << "/" - << size() << std::endl; - std::cout << "Overall correctness: " - << (get_correctness_passed() ? "PASSED" : "FAILED") << std::endl; - } else { - std::cout << "No successful test cases to report" << std::endl; - } -} - -float TestResult::get_total_avg_time_us() const { - if (results_.empty()) { - return 0.0f; - } - - float sum = 0.0f; - size_t count = 0; - - for (const auto& result : results_) { - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - if (vulkan_execute_succeeded) { - sum += result.get_avg_time_us(); - count++; - } - } - - return count > 0 ? sum / count : 0.0f; -} - -float TestResult::get_total_gflops() const { - return gflops_; -} - -size_t TestResult::get_passed_count() const { - size_t count = 0; - for (const auto& result : results_) { - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - if (vulkan_execute_succeeded) { - count++; - } - } - return count; -} - -size_t TestResult::get_failed_count() const { - return results_.size() - get_passed_count(); -} - -const BenchmarkResult* TestResult::get_fastest_result() const { - const BenchmarkResult* fastest = nullptr; - - for (const auto& result : results_) { - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - if (vulkan_execute_succeeded) { - if (!fastest || result.get_avg_time_us() < fastest->get_avg_time_us()) { - fastest = &result; - } - } - } - - return fastest; -} - -const BenchmarkResult* TestResult::get_slowest_result() const { - const BenchmarkResult* slowest = nullptr; - - for (const auto& result : results_) { - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - if (vulkan_execute_succeeded) { - if (!slowest || result.get_avg_time_us() > slowest->get_avg_time_us()) { - slowest = &result; - } - } - } - - return slowest; -} - -const BenchmarkResult* TestResult::get_highest_gflops_result() const { - // Since GFLOPS is now a TestResult-level metric rather than per-case, - // this method returns the fastest result as a proxy for highest performance - return get_fastest_result(); -} - -// Default FLOP calculation function (assumes 1 FLOP per element) -int64_t default_flop_calculator(const TestCase& test_case) { - // Calculate total elements from the first input tensor - int64_t total_elements = 1; - if (!test_case.empty() && test_case.num_inputs() > 0 && - test_case.inputs()[0].is_tensor()) { - const auto& sizes = test_case.inputs()[0].get_tensor_sizes(); - for (int64_t size : sizes) { - total_elements *= size; - } - } - - // Assume 1 FLOP per element for basic operations - return total_elements; -} - -BenchmarkGraph setup_compute_graph( - TestCase& test_case, - std::string op_name, - int op_invocations_per_execute) { - GraphConfig config; - config.enable_querypool = true; - // Pool sizing takes max(execute, prepack) * factor, so scaling the factor - // also over-reserves the prepack side (encoded once). Accepted: precise - // execute-only sizing would need runtime changes. - config.descriptor_pool_safety_factor *= - std::max(1, op_invocations_per_execute); - // Default-on (opt-out via TestCase::set_force_resize(false)): force every - // DynamicDispatchNode to run its resize function when execute_test_case - // runs propagate_resize() after prepack, exercising the op's resize formula - // even when input shapes are unchanged. - config.force_resize = test_case.get_force_resize(); - config.force_narrow_int4_tile = test_case.get_force_narrow_int4_tile(); - auto graph = std::make_unique(config); - - std::vector input_values; - - // Process input ValueSpecs - for (size_t i = 0; i < test_case.num_inputs(); ++i) { - const ValueSpec& input_spec = test_case.inputs()[i]; - - if (input_spec.is_none()) { - input_values.push_back(graph->add_none()); - } else if (input_spec.is_float()) { - ValueRef input_value = - graph->add_scalar(static_cast(input_spec.get_float_value())); - input_values.push_back(input_value); - } else if (input_spec.is_int()) { - ValueRef input_value = - graph->add_scalar(static_cast(input_spec.get_int_value())); - input_values.push_back(input_value); - } else if (input_spec.is_bool()) { - ValueRef input_value = graph->add_scalar(input_spec.get_bool_value()); - input_values.push_back(input_value); - } else if (input_spec.is_int_list()) { - // Convert int32_t list to int64_t list for ComputeGraph - const auto& int32_list = input_spec.get_int_list(); - std::vector int64_list; - int64_list.reserve(int32_list.size()); - for (int32_t val : int32_list) { - int64_list.push_back(static_cast(val)); - } - ValueRef input_value = graph->add_scalar_list(std::move(int64_list)); - input_values.push_back(input_value); - } else if (input_spec.is_string()) { - std::string str_copy = input_spec.get_string_value(); - ValueRef input_value = graph->add_string(std::move(str_copy)); - input_values.push_back(input_value); - } else if (input_spec.is_constant()) { - ValueRef input_value = graph->add_tensorref( - input_spec.get_tensor_sizes(), - input_spec.dtype, - input_spec.get_data_ptr()); - input_values.push_back(input_value); - } else { - IOValueRef input_io = graph->add_input_tensor( - input_spec.get_tensor_sizes(), - input_spec.dtype, - input_spec.storage_type, - input_spec.memory_layout); - input_values.push_back(input_io.value); - } - } - - std::vector output_values; - - // Process output ValueSpecs - for (size_t i = 0; i < test_case.num_outputs(); ++i) { - const ValueSpec& output_spec = test_case.outputs()[i]; - - if (!output_spec.is_tensor()) { - throw std::runtime_error("All output specifications must be tensors"); - } - - // Create output tensor - ValueRef output_value = graph->add_tensor( - output_spec.get_tensor_sizes(), - output_spec.dtype, - output_spec.storage_type, - output_spec.memory_layout); - - output_values.push_back(output_value); - } - - // Get the operator function and call it - auto opFn = VK_GET_OP_FN(op_name); - - // Create arguments vector for the operator function - std::vector op_args = input_values; - op_args.insert(op_args.end(), output_values.begin(), output_values.end()); - - // Nodes added before the op are staging uploads; nodes added after are - // staging downloads. Only the op's own nodes are repeated by benchmarks. - const size_t op_begin = graph->execute_nodes().size(); - opFn(*graph, op_args); - const size_t op_end = graph->execute_nodes().size(); - - for (size_t i = 0; i < output_values.size(); ++i) { - graph->set_output_value(output_values[i]); - } - return {std::move(graph), {op_begin, op_end}}; -} - -// Test execution utilities -BenchmarkResult execute_test_case( - TestCase& test_case, - int warmup_runs, - int benchmark_runs, - int chained_dispatches, - bool write_outputs) { - BenchmarkResult result( - test_case.name().empty() ? "unnamed_test_case" : test_case.name()); - - // Initialize querypool if using GPU timestamps - if (use_gpu_timestamps()) { - api::context()->initialize_querypool(); - } - - // Build the operator once. Benchmark repetition is encoded separately so - // persistent graph allocations are not duplicated. - BenchmarkGraph benchmark = setup_compute_graph( - test_case, test_case.operator_name(), chained_dispatches); - ComputeGraph& graph = *benchmark.graph; - - // Prepare the graph - graph.prepare(); - graph.prepack(); - - // Run resize functions once so force_resize exercises resize formulas even - // though the record/replay path below never calls propagate_resize(). - graph.propagate_resize(); - - // Copy input data into the graph's staging buffers - size_t graph_input_idx = 0; - for (size_t i = 0; i < test_case.num_inputs(); ++i) { - const ValueSpec& input_spec = test_case.inputs()[i]; - - // Only non-constant tensor inputs correspond to graph.inputs() entries - bool is_graph_input = input_spec.is_tensor() && !input_spec.is_constant() && - !input_spec.is_none(); - if (!is_graph_input) { - continue; - } - - if (graph_input_idx < graph.inputs().size()) { - const auto& input_ref = graph.inputs()[graph_input_idx]; - - // Get the appropriate data based on dtype - const void* data_ptr = nullptr; - size_t data_numel = input_spec.numel(); - - switch (input_spec.dtype) { - case vkapi::kFloat: - data_ptr = input_spec.get_float_data().data(); - break; - case vkapi::kHalf: - data_ptr = input_spec.get_half_data().data(); - break; - case vkapi::kInt: - data_ptr = input_spec.get_int32_data().data(); - break; - case vkapi::kChar: - data_ptr = input_spec.get_int8_data().data(); - break; - case vkapi::kByte: - data_ptr = input_spec.get_uint8_data().data(); - break; - default: - throw std::runtime_error("Unsupported data type for input tensor"); - } - - // Copy data into staging buffer - graph.maybe_cast_and_copy_into_staging( - input_ref.staging, data_ptr, data_numel, input_spec.dtype); - } - ++graph_input_idx; - } - - RepeatedGraphExecutor graph_executor( - graph, chained_dispatches, benchmark.op_nodes); - - // Warmup runs - for (int run = 0; run < warmup_runs; ++run) { - graph_executor.execute(); - } - - // Benchmark runs - collect individual iteration timings - float total_cpu_time_us = 0.0f; - float total_gpu_time_us = 0.0f; - - const float chained_dispatches_f = static_cast(chained_dispatches); - - for (int run = 0; run < benchmark_runs; ++run) { - // Measure CPU time for each execute() call - auto cpu_start = std::chrono::high_resolution_clock::now(); - graph_executor.execute(); - auto cpu_end = std::chrono::high_resolution_clock::now(); - - auto cpu_duration = std::chrono::duration_cast( - cpu_end - cpu_start); - // Each execute() ran chained_dispatches op invocations; report - // per-invocation time. - float cpu_time_us = - static_cast(cpu_duration.count()) / chained_dispatches_f; - total_cpu_time_us += cpu_time_us; - - // Collect per-shader GPU timing - get raw shader results to preserve - // metadata - graph.context()->querypool().extract_results(); - const auto shader_results = - graph.context()->querypool().get_shader_timestamp_data(); - - // Calculate total GPU time from per-shader timings - float gpu_time_us = 0.0f; - for (const auto& shader_result : shader_results) { - // Check if this shader matches any filter pattern - bool filtered = false; - for (const auto& filter_pattern : test_case.get_shader_filter()) { - if (shader_result.kernel_name.find(filter_pattern) != - std::string::npos) { - filtered = true; - break; - } - } - - if (!filtered) { - uint64_t duration_ns = - shader_result.end_time_ns - shader_result.start_time_ns; - float duration_us = static_cast(duration_ns) / 1000.0f; - gpu_time_us += duration_us; - // Per-shader sample is already one querypool entry per dispatch (at - // chained_dispatches>1 we get that many entries per shader), so do NOT - // divide here — the aggregator (get_avg_time_us) averages across them - // naturally. - result.add_shader_timing( - shader_result.kernel_name, - duration_us, - shader_result.metadata.gwg, - shader_result.metadata.lwg); - } - } - // gpu_time_us aggregates chained_dispatches worth of shader runs; divide - // it down to per-invocation time. - gpu_time_us /= chained_dispatches_f; - total_gpu_time_us += gpu_time_us; - - // Add the appropriate timing based on the flag - float iter_time_us = use_gpu_timestamps() ? gpu_time_us : cpu_time_us; - result.add_iter_timing(iter_time_us); - } - - // Calculate averages for display - float avg_cpu_time_us = total_cpu_time_us / benchmark_runs; - float avg_gpu_time_us = total_gpu_time_us / benchmark_runs; - - // Print both timings if latency printing is enabled - if (print_latencies()) { - if (use_gpu_timestamps()) { - graph.context()->querypool().print_results(); - } - std::cout << " CPU timing: " << std::fixed << std::setprecision(3) - << avg_cpu_time_us << " μs" << std::endl; - std::cout << " GPU timing: " << std::fixed << std::setprecision(3) - << avg_gpu_time_us << " μs" << std::endl; - std::cout << " Using " << (use_gpu_timestamps() ? "GPU" : "CPU") - << " timing for result" << std::endl; - } - - // Copy output data from the graph's staging buffers. The benchmarking path - // skips this work since it doesn't need correctness data — the probe pass - // has already populated test_case.outputs() at chained_dispatches=1. - if (write_outputs) { - for (size_t i = 0; i < test_case.num_outputs(); ++i) { - ValueSpec& output_spec = test_case.outputs()[i]; - - if (output_spec.is_tensor() && i < graph.outputs().size()) { - const auto& output_ref = graph.outputs()[i]; - - // Ensure output data vector is properly sized - size_t data_numel = output_spec.numel(); - output_spec.resize_data(data_numel); - - // Get mutable data pointer for the output - void* data_ptr = output_spec.get_mutable_data_ptr(); - - if (data_ptr != nullptr) { - // Copy data from staging buffer to output spec - graph.maybe_cast_and_copy_from_staging( - output_ref.staging, data_ptr, data_numel, output_spec.dtype); - } - - // Print output tensor data if output printing is enabled - if (print_output()) { - std::string output_name = "Output[" + std::to_string(i) + "]"; - print_valuespec_data(output_spec, output_name); - } - } - } - } - - return result; -} - -TestResult execute_test_cases( - std::function()> test_case_generator, - FlopCalculatorFunc flop_calculator, - const std::string& operation_name, - int warmup_runs, - int benchmark_runs, - ReferenceComputeFunc reference_compute_func) { - TestResult results(operation_name); - - // Generate all test cases - std::vector test_cases = test_case_generator(); - - std::cout << "Executing " << test_cases.size() << " test cases for " - << operation_name << std::endl; - print_separator(); - - // Group test cases by ReferenceKey for caching reference computations - // Use a vector to preserve the order in which groups first appear - std::vector group_order; - std::unordered_map, ReferenceKeyHash> - groups; - for (size_t i = 0; i < test_cases.size(); ++i) { - ReferenceKey key = ReferenceKey::from_test_case(test_cases[i]); - if (groups.find(key) == groups.end()) { - group_order.push_back(key); - } - groups[key].push_back(i); - } - - bool any_correctness_failed = false; - float total_gflops = 0.0f; - size_t test_case_counter = 0; - - // Process each group: generate data, compute reference, execute, and print - // Iterate in the order groups first appeared in test_cases - for (const auto& key : group_order) { - const auto& indices = groups[key]; - if (indices.empty()) - continue; - - // Get first test case as the "prototype" - size_t prototype_idx = indices[0]; - TestCase& prototype = test_cases[prototype_idx]; - - // Generate data for prototype with deterministic seed based on key - int group_seed = - static_cast(std::hash{}(key.key_string) % 10000); - for (auto& input : prototype.inputs()) { - input.ensure_data_generated(group_seed++); - } - - // Compute reference once for prototype - bool ref_computed = false; - if (reference_compute_func) { - try { - reference_compute_func(prototype); - ref_computed = true; - } catch (const std::invalid_argument&) { - // Reference computation skipped for this group - } - } - - // Copy data and reference to other test cases in group - for (size_t i = 1; i < indices.size(); ++i) { - size_t tc_idx = indices[i]; - TestCase& tc = test_cases[tc_idx]; - - // Copy input data from prototype - for (size_t j = 0; - j < tc.inputs().size() && j < prototype.inputs().size(); - ++j) { - auto& dest = tc.inputs()[j]; - const auto& src = prototype.inputs()[j]; - if (dest.is_tensor() && src.is_tensor() && dest.sizes == src.sizes && - dest.dtype == src.dtype) { - dest.share_data_from(src); - } - } - - // Copy reference output data if available - if (ref_computed) { - for (size_t j = 0; - j < tc.outputs().size() && j < prototype.outputs().size(); - ++j) { - const auto& src = prototype.outputs()[j]; - auto& dest = tc.outputs()[j]; - if (dest.is_tensor() && src.is_tensor() && dest.sizes == src.sizes && - dest.dtype == src.dtype) { - dest.share_reference_from(src); - } - } - } - } - - // Execute and print results for all test cases in this group - for (size_t tc_idx : indices) { - TestCase& test_case = test_cases[tc_idx]; - ++test_case_counter; - - // Execute single test case - BenchmarkResult result; - bool shader_not_supported = false; - try { - // Always run a probe pass at chained_dispatches=1 with - // write_outputs=true. This populates test_case.outputs() for the - // downstream correctness check with a clean single-dispatch result, - // and also gives us probe_us for adaptive sizing of the measurement - // run's chained_dispatches. - // - // probe_then_scale (Google Benchmark style): size chained_dispatches - // so each measurement execute() takes ~target_us. Tiny ops get a - // large factor (driving GPU governor escalation); heavy ops get a - // small factor (bounded wall time). Caveat: on Adreno 740 the probe - // runs at the DCVS-pinned clock (~220 MHz), so probe_us is inflated - // and the computed factor comes out under-sized for boost clock — but - // the default target is generous enough that an under-sized factor - // still drives sustained activity. - BenchmarkResult probe_result = execute_test_case( - test_case, - /*warmup_runs=*/1, - /*benchmark_runs=*/1, - /*chained_dispatches=*/1, - /*write_outputs=*/true); - float probe_us = probe_result.get_avg_time_us(); - if (probe_us < kMinProbeTimeUs) { - probe_us = kMinProbeTimeUs; - } - - int chained_dispatches; - const int manual_n = test_case.get_op_invocations_per_execute(); - if (manual_n > 0) { - chained_dispatches = manual_n; - } else { - const int target_us = test_case.get_target_execute_time_us(); - chained_dispatches = - std::max(1, static_cast(target_us / probe_us)); - chained_dispatches = - std::min(chained_dispatches, kMaxChainedDispatches); - } - if (debugging()) { - std::cout << "[probe] " << test_case.name() - << ": probe_us=" << probe_us - << ", chained_dispatches=" << chained_dispatches - << std::endl; - } - - // Measurement pass: chained_dispatches stacking, skip output copy - // since the probe already wrote the correctness data. - result = execute_test_case( - test_case, - warmup_runs, - benchmark_runs, - chained_dispatches, - /*write_outputs=*/false); - result.set_operator_name(test_case.operator_name()); - } catch (const vkcompute::vkapi::ShaderNotSupportedError&) { - result = BenchmarkResult( - test_case.name().empty() ? "unnamed_test_case" : test_case.name(), - test_case.operator_name()); - shader_not_supported = true; - } - - // Determine if this test case passed (has valid timing data) - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - - if (shader_not_supported) { - result.set_correctness_status(CorrectnessStatus::SKIPPED); - } else if (!vulkan_execute_succeeded) { - result.set_correctness_status(CorrectnessStatus::FAILED); - } else if (!ref_computed) { - result.set_correctness_status(CorrectnessStatus::SKIPPED); - } else { - // Reference function provided and succeeded - validate outputs - bool correctness_passed = true; - - for (size_t output_idx = 0; output_idx < test_case.num_outputs(); - ++output_idx) { - const ValueSpec& output_spec = test_case.outputs()[output_idx]; - - if (!output_spec.validate_against_reference( - test_case.get_abs_tolerance(), - test_case.get_rel_tolerance())) { - correctness_passed = false; - std::cout << " Correctness validation FAILED for test " - << result.get_kernel_name() << std::endl; - print_valuespec_data(output_spec, "vulkan output"); - print_valuespec_data(output_spec, "ref output", true); - } - } - - if (correctness_passed) { - result.set_correctness_status(CorrectnessStatus::PASSED); - } else { - any_correctness_failed = true; - result.set_correctness_status(CorrectnessStatus::FAILED); - } - } - - // Calculate GFLOPS for this test case using the provided FLOP calculator - float case_gflops = 0.0f; - if (vulkan_execute_succeeded) { - // Use the provided FLOP calculator to get total FLOPs for this test - // case - int64_t total_flops = flop_calculator(test_case); - float flops = static_cast(total_flops); - float avg_time_us = result.get_avg_time_us(); - if (avg_time_us > 0.0f && total_flops > 0) { - case_gflops = (flops / 1e9f) / (avg_time_us / 1e6f); - } - - total_gflops += case_gflops; - } else { - case_gflops = -1.0f; // Indicate failure - } - - // Calculate tensor info for display - std::string size_info = "["; - if (!test_case.empty() && test_case.num_inputs() > 0 && - test_case.inputs()[0].is_tensor()) { - const auto& sizes = test_case.inputs()[0].get_tensor_sizes(); - for (size_t j = 0; j < sizes.size(); ++j) { - size_info += std::to_string(sizes[j]); - if (j < sizes.size() - 1) - size_info += "x"; - } - } - size_info += "]"; - - // Print progress using the BenchmarkResult member function - result.print_summary(test_case_counter, size_info, case_gflops); - - // Add result to collection - results.add_result(std::move(result)); - - test_case.clear(); - } - } - - // Set the overall results on the TestResult - results.set_correctness_passed(!any_correctness_failed); - results.set_gflops(total_gflops); - - print_separator(); - std::cout << "Completed " << results.size() << " test cases" << std::endl; - - // Hard-fail if any correctness check failed. The per-case loop above keeps - // running after a mismatch (so the full pass/fail matrix and per-16x16-tile - // mismatch maps are printed for debugging) rather than aborting on the first - // failure; throwing here preserves the original abort-on-mismatch behavior. - if (any_correctness_failed) { - throw std::runtime_error("Correctness validation failed"); - } - - return results; -} - -// Convenience overload that uses the default FLOP calculator -TestResult execute_test_cases( - std::function()> test_case_generator, - const std::string& operation_name, - int warmup_runs, - int benchmark_runs, - ReferenceComputeFunc reference_compute_func) { - return execute_test_cases( - test_case_generator, - default_flop_calculator, - operation_name, - warmup_runs, - benchmark_runs, - reference_compute_func); -} - -// Utility functions for printing -void print_performance_header() { - std::cout << "\n=== Compute Shader Performance Benchmark ===" << std::endl; -} - -void print_separator() { - std::cout << std::string(70, '-') << std::endl; -} - -// ValueSpec data printing utilities -void print_valuespec_data( - const ValueSpec& spec, - const std::string& name, - const bool print_ref_data, - size_t max_elements, - int precision) { - std::cout << "\n" << name << " Data:" << std::endl; - std::cout << " Type: " << spec.to_string() << std::endl; - - if (!spec.is_tensor()) { - if (spec.is_int()) { - std::cout << " Value: " << spec.get_int_value() << std::endl; - } else if (spec.is_int_list()) { - const auto& int_list = spec.get_int_list(); - std::cout << " Values: ["; - size_t print_count = std::min(max_elements, int_list.size()); - for (size_t i = 0; i < print_count; ++i) { - std::cout << int_list[i]; - if (i < print_count - 1) - std::cout << ", "; - } - if (int_list.size() > max_elements) { - std::cout << ", ... (" << (int_list.size() - max_elements) << " more)"; - } - std::cout << "]" << std::endl; - } - return; - } - - // Print tensor data - size_t total_elements = spec.numel(); - size_t print_count = std::min(max_elements, total_elements); - - std::cout << " Total elements: " << total_elements << std::endl; - std::cout << " Data (first " << print_count << " elements): ["; - - std::cout << std::fixed << std::setprecision(precision); - - switch (spec.dtype) { - case vkapi::kFloat: { - auto data = spec.get_float_data().data(); - if (print_ref_data) { - data = spec.get_ref_float_data().data(); - } - for (size_t i = 0; i < print_count; ++i) { - std::cout << data[i]; - if (i < print_count - 1) - std::cout << ", "; - } - break; - } - case vkapi::kHalf: { - if (print_ref_data) { - const auto& ref = spec.get_ref_float_data(); - for (size_t i = 0; i < print_count; ++i) { - std::cout << ref[i]; - if (i < print_count - 1) - std::cout << ", "; - } - break; - } - const auto& data = spec.get_half_data(); - for (size_t i = 0; i < print_count; ++i) { - // Convert IEEE 754 half-precision bit pattern back to float. - float value = half_to_float(data[i]); - std::cout << value; - if (i < print_count - 1) - std::cout << ", "; - } - break; - } - case vkapi::kInt: { - const auto& data = spec.get_int32_data(); - for (size_t i = 0; i < print_count; ++i) { - std::cout << data[i]; - if (i < print_count - 1) - std::cout << ", "; - } - break; - } - case vkapi::kChar: { - const auto& data = spec.get_int8_data(); - if (spec.is_int4()) { - // Print each 4-bit value individually - size_t element_count = 0; - for (size_t i = 0; i < data.size() && element_count < print_count; - ++i) { - // Extract lower 4 bits (signed) - int8_t lower_4bits = data[i] & 0x0F; - if (lower_4bits > 7) - lower_4bits -= 16; // Convert to signed - std::cout << static_cast(lower_4bits); - element_count++; - - if (element_count < print_count) { - std::cout << ", "; - // Extract upper 4 bits (signed) - int8_t upper_4bits = (data[i] >> 4) & 0x0F; - if (upper_4bits > 7) - upper_4bits -= 16; // Convert to signed - std::cout << static_cast(upper_4bits); - element_count++; - - if (element_count < print_count) - std::cout << ", "; - } - } - } else { - for (size_t i = 0; i < print_count; ++i) { - std::cout << static_cast(data[i]); - if (i < print_count - 1) - std::cout << ", "; - } - } - break; - } - case vkapi::kByte: { - const auto& data = spec.get_uint8_data(); - if (spec.is_int4()) { - // Print each 4-bit value individually - size_t element_count = 0; - for (size_t i = 0; i < data.size() && element_count < print_count; - ++i) { - // Extract lower 4 bits - uint8_t lower_4bits = data[i] & 0x0F; - std::cout << static_cast(lower_4bits); - element_count++; - - if (element_count < print_count) { - std::cout << ", "; - // Extract upper 4 bits - uint8_t upper_4bits = (data[i] >> 4) & 0x0F; - std::cout << static_cast(upper_4bits); - element_count++; - - if (element_count < print_count) - std::cout << ", "; - } - } - } else { - for (size_t i = 0; i < print_count; ++i) { - std::cout << static_cast(data[i]); - if (i < print_count - 1) - std::cout << ", "; - } - } - break; - } - default: - std::cout << "unsupported data type"; - break; - } - - if (total_elements > max_elements) { - std::cout << ", ... (" << (total_elements - max_elements) << " more)"; - } - std::cout << "]" << std::endl; - - // Print some statistics for tensor data - if (total_elements > 0) { - float min_val = 0.0f, max_val = 0.0f, sum = 0.0f; - bool first = true; - - for (size_t i = 0; i < total_elements; ++i) { - float val = spec.get_element(i); - if (first) { - min_val = max_val = val; - first = false; - } else { - min_val = std::min(min_val, val); - max_val = std::max(max_val, val); - } - sum += val; - } - - float mean = sum / total_elements; - std::cout << " Statistics: min=" << std::setprecision(precision) << min_val - << ", max=" << max_val << ", mean=" << mean << ", sum=" << sum - << std::endl; - } -} - -ValueRef quantized_weights_canvas( - ComputeGraph& graph, - const ValueRef weight_ref) { - const auto original_sizes = graph.sizes_of(weight_ref); - - std::vector sorted_sizes = original_sizes; - std::sort(sorted_sizes.begin(), sorted_sizes.end(), std::greater()); - int64_t largest1 = sorted_sizes.size() > 0 ? sorted_sizes[0] : 0; - - std::vector final_sizes = {1, largest1, largest1}; - - // Debug logging if debugging flag is set - if (debugging()) { - std::cout << "Debug: Creating quantized weights canvas tensor" << std::endl; - std::cout << "Debug: Original sizes: ["; - for (size_t i = 0; i < original_sizes.size(); ++i) { - std::cout << original_sizes[i]; - if (i < original_sizes.size() - 1) - std::cout << ", "; - } - std::cout << "]" << std::endl; - std::cout << "Debug: Canvas sizes: ["; - for (size_t i = 0; i < final_sizes.size(); ++i) { - std::cout << final_sizes[i]; - if (i < final_sizes.size() - 1) - std::cout << ", "; - } - std::cout << "]" << std::endl; - } - - ValueRef packed_weight = graph.add_tensor( - final_sizes, vkapi::kInt, utils::kTexture3D, utils::kWidthPacked); - - std::string kernel_name = "packed_int32_canvas"; - add_storage_type_suffix(kernel_name, graph.storage_type_of(packed_weight)); - - graph.prepack_nodes().emplace_back(new PrepackNode( - graph, - VK_KERNEL_FROM_STR(kernel_name), - graph.create_gwg(packed_weight), - graph.create_lwg(packed_weight), - weight_ref, - packed_weight, - // UBOs - {graph.logical_limits_ubo(packed_weight)}, - // Specialization constants - {}, - // Push Constants - {})); - - return packed_weight; -} - -ValueRef float_tensor_canvas(ComputeGraph& graph, const ValueRef weight_ref) { - const auto original_sizes = graph.sizes_of(weight_ref); - - std::vector sorted_sizes = original_sizes; - std::sort(sorted_sizes.begin(), sorted_sizes.end(), std::greater()); - int64_t largest1 = sorted_sizes.size() > 0 ? sorted_sizes[0] : 0; - - std::vector final_sizes = {1, largest1, largest1}; - - // Debug logging if debugging flag is set - if (debugging()) { - std::cout << "Debug: Creating float tensor canvas" << std::endl; - std::cout << "Debug: Original sizes: ["; - for (size_t i = 0; i < original_sizes.size(); ++i) { - std::cout << original_sizes[i]; - if (i < original_sizes.size() - 1) - std::cout << ", "; - } - std::cout << "]" << std::endl; - std::cout << "Debug: Canvas sizes: ["; - for (size_t i = 0; i < final_sizes.size(); ++i) { - std::cout << final_sizes[i]; - if (i < final_sizes.size() - 1) - std::cout << ", "; - } - std::cout << "]" << std::endl; - } - - ValueRef packed_weight = graph.add_tensor( - final_sizes, vkapi::kFloat, utils::kTexture3D, utils::kWidthPacked); - - graph.prepack_nodes().emplace_back(new PrepackNode( - graph, - VK_KERNEL_FROM_STR("float_canvas"), - graph.create_gwg(packed_weight), - graph.create_lwg(packed_weight), - weight_ref, - packed_weight, - // UBOs - {graph.logical_limits_ubo(packed_weight)}, - // Specialization constants - {}, - // Push Constants - {})); - - return packed_weight; -} - -// Compute weight sums for quantized operations (linear and convolution) -void compute_weight_sums( - ValueSpec& weight_sums, - const ValueSpec& quantized_weight, - int64_t out_features, - int64_t elements_per_output_feature) { - auto& weight_sums_data = weight_sums.get_int32_data(); - auto& quantized_weight_data = quantized_weight.get_int8_data(); - - // Don't resize down - the buffer may be pre-allocated with aligned size. - // Only resize up if needed. - if (weight_sums_data.size() < static_cast(out_features)) { - weight_sums_data.resize(out_features); - } - - // For each output feature, compute the sum of quantized weights - for (int64_t out_f = 0; out_f < out_features; ++out_f) { - int32_t sum = 0; - for (int64_t elem = 0; elem < elements_per_output_feature; ++elem) { - // Weight indexing depends on the layout: - // For linear: [out_features, in_features] -> out_f * - // elements_per_output_feature + elem For conv2d: [C_out, C_in * K_h * - // K_w] -> out_f * elements_per_output_feature + elem - int64_t weight_idx = out_f * elements_per_output_feature + elem; - sum += static_cast(quantized_weight_data[weight_idx]); - } - weight_sums_data[out_f] = sum; - } -} - -// Compute weight sums for 4D quantized conv2d operations -// Weight layout: [C_out, K_h, K_w, align_up_4(C_in_per_group)] -void compute_weight_sums_4d( - ValueSpec& weight_sums, - const ValueSpec& quantized_weight, - int64_t out_channels, - int64_t kernel_h, - int64_t kernel_w, - int64_t aligned_in_channels) { - auto& weight_sums_data = weight_sums.get_int32_data(); - auto& quantized_weight_data = quantized_weight.get_int8_data(); - - weight_sums_data.resize(out_channels); - - // For each output channel, compute the sum of quantized weights - for (int64_t out_c = 0; out_c < out_channels; ++out_c) { - int32_t sum = 0; - - for (int64_t kh = 0; kh < kernel_h; ++kh) { - for (int64_t kw = 0; kw < kernel_w; ++kw) { - for (int64_t in_c = 0; in_c < aligned_in_channels; ++in_c) { - // Weight indexing: [out_c, kh, kw, in_c] - int64_t weight_idx = - out_c * (kernel_h * kernel_w * aligned_in_channels) + - kh * (kernel_w * aligned_in_channels) + kw * aligned_in_channels + - in_c; - sum += static_cast(quantized_weight_data[weight_idx]); - } - } - } - - weight_sums_data[out_c] = sum; - } -} - -// Helper function to unpack 4-bit values from uint8 (same as in -// q4gsw_linear.cpp) -std::pair unpack_4bit_utils(uint8_t packed) { - // Extract lower 4 bits and upper 4 bits - int8_t lower = packed & 0x0F; - int8_t upper = (packed >> 4) & 0x0F; - - // Subtract 8 from unpacked 4-bit values - lower -= 8; - upper -= 8; - - return std::make_pair(lower, upper); -} - -// Compute weight sums for 4-bit group symmetric quantized weights -void compute_weight_sums_4bit_grouped( - ValueSpec& weight_sums, - const ValueSpec& quantized_weight, - int64_t num_groups, - int64_t out_features, - int64_t group_size) { - auto& weight_sums_data = weight_sums.get_int32_data(); - auto& quantized_weight_data = quantized_weight.get_uint8_data(); - - // Resize to [num_groups, out_features] - weight_sums_data.resize(num_groups * out_features); - - // For each group and each output feature, compute the sum of quantized - // weights in that group - for (int64_t group_idx = 0; group_idx < num_groups; ++group_idx) { - for (int64_t out_f = 0; out_f < out_features; ++out_f) { - int32_t sum = 0; - - // Sum weights for this group and output feature - for (int64_t in_group = 0; in_group < group_size; ++in_group) { - int64_t in_f = group_idx * group_size + in_group; - - // Get packed weight value - weight matrix is [N, K/2] - int64_t weight_idx = - out_f * ((num_groups * group_size) / 2) + (in_f / 2); - uint8_t packed_weight = quantized_weight_data[weight_idx]; - - // Unpack 4-bit weight - auto unpacked = unpack_4bit_utils(packed_weight); - int8_t weight_4bit = (in_f % 2 == 0) ? unpacked.first : unpacked.second; - - sum += static_cast(weight_4bit); - } - - // Store sum for this group and output feature - int64_t sums_idx = group_idx * out_features + out_f; - weight_sums_data[sums_idx] = sum; - } - } -} - -} // namespace prototyping -} // namespace vulkan -} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/utils.h b/backends/vulkan/test/custom_ops/utils.h deleted file mode 100644 index b94d0d41421..00000000000 --- a/backends/vulkan/test/custom_ops/utils.h +++ /dev/null @@ -1,1094 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#pragma once - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace executorch { -namespace vulkan { -namespace prototyping { - -using namespace vkcompute; - -// -// ReferenceKey for caching reference computations -// - -// Captures the identity of input conditions for test case grouping. -// Test cases with the same ReferenceKey should produce identical reference -// outputs, so reference computation can be cached and reused. -struct ReferenceKey { - std::string key_string; - - static ReferenceKey from_test_case(const class TestCase& tc); - - bool operator==(const ReferenceKey& other) const { - return key_string == other.key_string; - } -}; - -struct ReferenceKeyHash { - size_t operator()(const ReferenceKey& k) const { - return std::hash{}(k.key_string); - } -}; - -// -// Global configuration options -// - -bool print_output(); -void set_print_output(bool print_output); - -bool print_latencies(); -void set_print_latencies(bool print_latencies); - -bool use_gpu_timestamps(); -void set_use_gpu_timestamps(bool use_timestamps); - -bool debugging(); -void set_debugging(bool enable_debugging); - -// -// Shader filter defaults -// - -// Default shader filter: excludes layout conversions and quantization overhead -// This is used for tests where quantize/dequantize are overhead operations -inline const std::vector kDefaultShaderFilter = { - "nchw_to", - "to_nchw", - "quantize_and_pack_4w4c", - "unpack_4w4c_and_dequantize"}; - -// Layout-only filter: only excludes layout conversions -// Use this for tests where quantize/dequantize ARE the operations being tested -inline const std::vector kLayoutOnlyShaderFilter = { - "nchw_to", - "to_nchw"}; - -// -// String utilities -// - -// Helper function to get abbreviated layout names for test case naming -inline std::string layout_abbrev(utils::GPUMemoryLayout layout) { - switch (layout) { - case utils::kWidthPacked: - return "WP"; - case utils::kChannelsPacked: - return "CP"; - case utils::kPackedInt8_4W: - return "4W"; - case utils::kPackedInt8_4C: - return "4C"; - case utils::kPackedInt8_4W4C: - return "4W4C"; - case utils::kPackedInt8_4H4W: - return "4H4W"; - case utils::kPackedInt8_4C1W: - return "4C1W"; - default: - return "UNK"; - } -} - -// Helper function to get abbreviated storage type names for test case naming -inline std::string storage_type_abbrev(utils::StorageType storage_type) { - switch (storage_type) { - case utils::kTexture3D: - return "Tex"; - case utils::kBuffer: - return "Buf"; - default: - return "UNK"; - } -} - -// Helper function to get combined storage type and layout representation -// Example: (kBuffer, kPackedInt8_4W4C) -> "Buf_4W4C" -inline std::string repr_str( - utils::StorageType storage_type, - utils::GPUMemoryLayout layout) { - return storage_type_abbrev(storage_type) + "(" + layout_abbrev(layout) + ")"; -} - -// Helper function to generate comma-separated shape string for test case naming -// Example: {1, 128, 56, 56} -> "1,128,56,56" -inline std::string shape_string(const std::vector& shape) { - std::string result; - for (size_t i = 0; i < shape.size(); ++i) { - if (i > 0) { - result += ","; - } - result += std::to_string(shape[i]); - } - return result; -} - -// Helper function to generate a bracketed shape string for test case naming -// Example: {1, 128, 56, 56} -> "[1,128,56,56]" -inline std::string shape_bracket(const std::vector& shape) { - return "[" + shape_string(shape) + "]"; -} - -// Short dtype symbol used in standardized test case labels -// Example: kFloat -> "f32", kHalf -> "f16", kChar -> "i8", kInt -> "i32" -inline std::string dtype_short(vkapi::ScalarType dtype) { - switch (dtype) { - case vkapi::kFloat: - return "f32"; - case vkapi::kHalf: - return "f16"; - case vkapi::kChar: - return "i8"; - case vkapi::kByte: - return "u8"; - case vkapi::kInt: - return "i32"; - case vkapi::kBool: - return "b"; - default: - return "?"; - } -} - -// Build a standardized test label of the form: -// " [ ]" -// where is "->" when the two differ, or just -// "" when they match. Sections are separated by two spaces. If -// suffix is non-empty it is appended after a single space (allowing callers -// to pass e.g. "[general]" or "[gemv] +bias"). -inline std::string make_test_label( - const std::string& prefix, - const std::string& in_dtype, - const std::string& out_dtype, - const std::string& shape_str, - const std::string& storage_str, - const std::string& suffix = "") { - const std::string dtype_str = - (in_dtype == out_dtype) ? in_dtype : in_dtype + "->" + out_dtype; - std::string label = - prefix + " " + dtype_str + " " + shape_str + " " + storage_str; - if (!suffix.empty()) { - label += " " + suffix; - } - return label; -} - -// -// ValueSpec class -// - -enum class SpecType { Tensor, IntList, Int, Float, Bool, String }; - -// Data generation types -enum class DataGenType { - FIXED, - RANDOM, - RANDOM_SCALES, - RANDINT, - RANDINT8, - RANDINT4, - ONES, - ONES_INT4, - ZEROS -}; - -// Value specification struct -struct ValueSpec { - std::vector sizes; - vkapi::ScalarType dtype; - utils::GPUMemoryLayout memory_layout; - utils::StorageType storage_type; - SpecType spec_type; - DataGenType data_gen_type; - bool is_constant_tensor; - bool is_none_flag; - bool is_int4_tensor; - std::string string_data; - - ValueSpec( - const std::vector& sizes, - vkapi::ScalarType dtype, - utils::StorageType storage_type = utils::kTexture3D, - utils::GPUMemoryLayout memory_layout = utils::kWidthPacked) - : sizes(sizes), - dtype(dtype), - memory_layout(memory_layout), - storage_type(storage_type), - spec_type(SpecType::Tensor), - data_gen_type(DataGenType::ZEROS), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(false) { - // Data generation is deferred until first access (any data getter or - // ensure_data_generated() triggers it). - } - - // Constructor for tensor with custom data generation type - ValueSpec( - const std::vector& sizes, - vkapi::ScalarType dtype, - utils::StorageType storage_type, - utils::GPUMemoryLayout memory_layout, - DataGenType data_gen_type) - : sizes(sizes), - dtype(dtype), - memory_layout(memory_layout), - storage_type(storage_type), - spec_type(SpecType::Tensor), - data_gen_type(data_gen_type), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(false) { - // Data generation is deferred until first access (any data getter or - // ensure_data_generated() triggers it). - } - - // Constructor for single int - ValueSpec(int32_t value) - : sizes({1}), - dtype(vkapi::kInt), - memory_layout(utils::kWidthPacked), - storage_type(utils::kTexture3D), - spec_type(SpecType::Int), - data_gen_type(DataGenType::FIXED), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(true) { - data_->int32_data.push_back(value); - } - - // Constructor for single float - ValueSpec(float value) - : sizes({1}), - dtype(vkapi::kFloat), - memory_layout(utils::kWidthPacked), - storage_type(utils::kTexture3D), - spec_type(SpecType::Float), - data_gen_type(DataGenType::FIXED), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(true) { - data_->float_data.push_back(value); - } - - // Constructor for single bool - ValueSpec(bool value) - : sizes({1}), - dtype(vkapi::kInt), - memory_layout(utils::kWidthPacked), - storage_type(utils::kTexture3D), - spec_type(SpecType::Bool), - data_gen_type(DataGenType::FIXED), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(true) { - data_->int32_data.push_back(value ? 1 : 0); - } - - // Constructor for int list - ValueSpec(const std::vector& values) - : sizes({static_cast(values.size())}), - dtype(vkapi::kInt), - memory_layout(utils::kWidthPacked), - storage_type(utils::kTexture3D), - spec_type(SpecType::IntList), - data_gen_type(DataGenType::FIXED), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(true) { - data_->int32_data = values; - } - - // Factory method for string (avoids ambiguity with vector constructor) - static ValueSpec make_string(const std::string& value) { - ValueSpec spec; - spec.sizes = {1}; - spec.dtype = vkapi::kInt; - spec.memory_layout = utils::kWidthPacked; - spec.storage_type = utils::kTexture3D; - spec.spec_type = SpecType::String; - spec.data_gen_type = DataGenType::FIXED; - spec.is_constant_tensor = false; - spec.is_none_flag = false; - spec.is_int4_tensor = false; - spec.data_generated_ = true; - spec.string_data = value; - return spec; - } - - // Default constructor - ValueSpec() - : dtype(vkapi::kFloat), - memory_layout(utils::kWidthPacked), - storage_type(utils::kTexture3D), - spec_type(SpecType::Tensor), - data_gen_type(DataGenType::ZEROS), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(false) {} - - int64_t numel() const; - size_t nbytes() const; - std::string to_string() const; - - bool is_tensor() const { - return spec_type == SpecType::Tensor; - } - bool is_int_list() const { - return spec_type == SpecType::IntList; - } - bool is_int() const { - return spec_type == SpecType::Int; - } - bool is_float() const { - return spec_type == SpecType::Float; - } - bool is_bool() const { - return spec_type == SpecType::Bool; - } - bool is_string() const { - return spec_type == SpecType::String; - } - - int32_t get_int_value() const { - ensure_data_generated(); - return data_->int32_data.empty() ? 0 : data_->int32_data[0]; - } - float get_float_value() const { - ensure_data_generated(); - return data_->float_data.empty() ? 0.0f : data_->float_data[0]; - } - bool get_bool_value() const { - ensure_data_generated(); - return data_->int32_data.empty() ? false : (data_->int32_data[0] != 0); - } - const std::string& get_string_value() const { - return string_data; - } - const std::vector& get_int_list() const { - ensure_data_generated(); - return data_->int32_data; - } - const std::vector& get_tensor_sizes() const { - return sizes; - } - - // References and pointers into tensor data must not be held across any other - // access to the same spec: a mutable access may detach the shared payload, - // leaving a previously returned reference bound to the old payload. Consume - // immediately. - const std::vector& get_float_data() const { - ensure_data_generated(); - return data_->float_data; - } - const std::vector& get_int32_data() const { - ensure_data_generated(); - return data_->int32_data; - } - const std::vector& get_half_data() const { - ensure_data_generated(); - return data_->half_data; - } - const std::vector& get_int8_data() const { - ensure_data_generated(); - return data_->int8_data; - } - const std::vector& get_uint8_data() const { - ensure_data_generated(); - return data_->uint8_data; - } - - std::vector& get_float_data() { - ensure_data_generated(); - ensure_unique_data(); - return data_->float_data; - } - std::vector& get_int32_data() { - ensure_data_generated(); - ensure_unique_data(); - return data_->int32_data; - } - std::vector& get_half_data() { - ensure_data_generated(); - ensure_unique_data(); - return data_->half_data; - } - std::vector& get_int8_data() { - ensure_data_generated(); - ensure_unique_data(); - return data_->int8_data; - } - std::vector& get_uint8_data() { - ensure_data_generated(); - ensure_unique_data(); - return data_->uint8_data; - } - - const std::vector& get_ref_float_data() const { - return reference_data_->float_data; - } - const std::vector& get_ref_int32_data() const { - return reference_data_->int32_data; - } - const std::vector& get_ref_half_data() const { - return reference_data_->half_data; - } - const std::vector& get_ref_int8_data() const { - return reference_data_->int8_data; - } - const std::vector& get_ref_uint8_data() const { - return reference_data_->uint8_data; - } - - std::vector& get_ref_float_data() { - ensure_unique_reference_data(); - return reference_data_->float_data; - } - std::vector& get_ref_int32_data() { - ensure_unique_reference_data(); - return reference_data_->int32_data; - } - std::vector& get_ref_half_data() { - ensure_unique_reference_data(); - return reference_data_->half_data; - } - std::vector& get_ref_int8_data() { - ensure_unique_reference_data(); - return reference_data_->int8_data; - } - std::vector& get_ref_uint8_data() { - ensure_unique_reference_data(); - return reference_data_->uint8_data; - } - - void resize_data(size_t new_size); - void* get_mutable_data_ptr(); - float get_element(size_t index) const; - - // Data generation methods for deferred generation and caching. - // - // ValueSpec is not thread-safe: lazy materialization and copy-on-write - // detach mutate shared state from const methods. Test cases are built and - // executed on a single thread. - // - // Implicit materialization (any data getter, resize_data) consumes the - // global seed counter. Callers needing deterministic data must call - // ensure_data_generated(explicit_seed) before any other access; a later - // seeded call is a no-op once data is generated. - bool is_data_generated() const { - return data_generated_; - } - void ensure_data_generated(int seed = -1) const; - void share_data_from(const ValueSpec& other); - void share_reference_from(const ValueSpec& other); - - // Set/get constant flag - bool is_constant() const { - return is_constant_tensor; - } - void set_constant(bool is_constant) { - is_constant_tensor = is_constant; - } - - // Set/get none flag - bool is_none() const { - return is_none_flag; - } - - void set_none(bool is_none) { - is_none_flag = is_none; - } - - // Set/get int4 flag - bool is_int4() const { - return is_int4_tensor; - } - void set_int4(bool is_int4) { - is_int4_tensor = is_int4; - } - - const void* get_data_ptr() const; - - // Correctness checking against reference data - // Returns true if computed data matches reference data within tolerance - // Only validates float tensors as specified in requirements - bool validate_against_reference( - float abs_tolerance = 2e-3f, - float rel_tolerance = 1e-3f) const; - - private: - struct TensorData { - std::vector float_data; - std::vector int32_data; - std::vector half_data; - std::vector int8_data; - std::vector uint8_data; - }; - - void ensure_unique_data() const; - void ensure_unique_reference_data() const; - void generate_tensor_data(int seed = -1) const; - - mutable bool data_generated_ = false; - mutable std::shared_ptr data_ = std::make_shared(); - mutable std::shared_ptr reference_data_ = - std::make_shared(); -}; - -// -// TestCase -// - -// Default per-execute() wall-clock target used by the probe-then-scale logic -// in execute_test_cases(). Picked generously enough that even an under-sized -// chained_dispatches factor (the probe runs at governor-pinned clock and -// underestimates the boost-clock latency) still drives sustained GPU -// activity during measurement. -constexpr int kDefaultTargetExecuteTimeUs = 100000; - -class TestCase { - public: - TestCase() - : abs_tolerance_(2e-3f), - rel_tolerance_(1e-3f), - shader_filter_(kDefaultShaderFilter) {} - TestCase(const std::string& name) - : name_(name), - abs_tolerance_(2e-3f), - rel_tolerance_(1e-3f), - shader_filter_(kDefaultShaderFilter) {} - - void set_name(const std::string& name) { - name_ = name; - } - const std::string& name() const { - return name_; - } - - void set_operator_name(const std::string& op_name) { - operator_name_ = op_name; - } - const std::string& operator_name() const { - return operator_name_; - } - - // Tolerance settings - void set_abs_tolerance(float abs_tolerance) { - abs_tolerance_ = abs_tolerance; - } - float get_abs_tolerance() const { - return abs_tolerance_; - } - - void set_rel_tolerance(float rel_tolerance) { - rel_tolerance_ = rel_tolerance; - } - float get_rel_tolerance() const { - return rel_tolerance_; - } - - // Shader filter settings - list of shader name patterns to exclude from - // timing - void set_shader_filter(const std::vector& filter) { - shader_filter_ = filter; - } - const std::vector& get_shader_filter() const { - return shader_filter_; - } - - // Manual override for the number of chained dispatches per measurement - // iteration (a.k.a. chained_dispatches). If > 0, the framework uses this - // directly and skips the probe phase. 0 (the default) means adaptive - // (probe-then-scale). - void set_op_invocations_per_execute(int n) { - op_invocations_per_execute_ = n; - } - int get_op_invocations_per_execute() const { - return op_invocations_per_execute_; - } - - // Target single-execute duration in microseconds. Used only when the - // manual chained_dispatches override is not set. Default - // kDefaultTargetExecuteTimeUs, picked generously enough to mitigate Adreno - // DCVS governor pinning during the probe. - void set_target_execute_time_us(int us) { - target_execute_time_us_ = us; - } - int get_target_execute_time_us() const { - return target_execute_time_us_; - } - - // When true, the ComputeGraph built for this test case sets - // GraphConfig::force_resize, so every DynamicDispatchNode runs its resize - // function once during measurement setup (execute_test_case runs - // propagate_resize() after prepack) even when no input shape changed. - // Because the output is already allocated at the swept shape, the resize - // must recompute the same shape from the current input — a wrong resize - // formula resizes the output to a mismatched shape and surfaces as a test - // failure. Default true (opt-out): every custom_ops test exercises its - // resize formulas across the swept shapes. Call set_force_resize(false) for - // the rare op whose resize fn is intentionally not shape-preserving under a - // fixed output allocation. - void set_force_resize(bool force_resize) { - force_resize_ = force_resize; - } - bool get_force_resize() const { - return force_resize_; - } - - // When true, the ComputeGraph sets GraphConfig::force_narrow_int4_tile, so - // the int4 linear picks its narrow (2-row) output tile on any GPU. That - // variant is otherwise reachable only on Mali, which no CI runner has. - void set_force_narrow_int4_tile(bool force) { - force_narrow_int4_tile_ = force; - } - bool get_force_narrow_int4_tile() const { - return force_narrow_int4_tile_; - } - - void add_input_spec(const ValueSpec& spec) { - inputs_.push_back(spec); - } - - const std::vector& inputs() const { - return inputs_; - } - - std::vector& inputs() { - return inputs_; - } - - size_t num_inputs() const { - return inputs_.size(); - } - - void add_output_spec(const ValueSpec& spec) { - outputs_.push_back(spec); - } - - const std::vector& outputs() const { - return outputs_; - } - - std::vector& outputs() { - return outputs_; - } - - size_t num_outputs() const { - return outputs_.size(); - } - - bool empty() const { - return inputs_.empty() && outputs_.empty(); - } - void clear() { - inputs_.clear(); - outputs_.clear(); - name_.clear(); - operator_name_.clear(); - abs_tolerance_ = 2e-3f; - rel_tolerance_ = 1e-3f; - shader_filter_ = kDefaultShaderFilter; - op_invocations_per_execute_ = 0; - target_execute_time_us_ = kDefaultTargetExecuteTimeUs; - force_resize_ = true; - force_narrow_int4_tile_ = false; - } - - private: - std::string name_; - std::string operator_name_; - std::vector inputs_; - std::vector outputs_; - float abs_tolerance_; - float rel_tolerance_; - std::vector shader_filter_; - int op_invocations_per_execute_ = 0; // 0 = adaptive - int target_execute_time_us_ = kDefaultTargetExecuteTimeUs; - bool force_resize_ = true; - bool force_narrow_int4_tile_ = false; -}; - -// -// BenchmarkResult -// - -enum class CorrectnessStatus { - SKIPPED, // No reference function provided - PASSED, // Reference function provided and validation passed - FAILED // Reference function provided but validation failed -}; - -// Per-shader timing data for detailed reporting -struct ShaderTiming { - std::string shader_name; - std::vector iter_timings_us; // Individual iteration timings - uint32_t gwg[3] = {0, 0, 0}; - uint32_t lwg[3] = {0, 0, 0}; - - float get_avg_time_us() const { - if (iter_timings_us.empty()) { - return 0.0f; - } - float sum = 0.0f; - for (float t : iter_timings_us) { - sum += t; - } - return sum / iter_timings_us.size(); - } -}; - -class BenchmarkResult { - public: - BenchmarkResult() : correctness_status_(CorrectnessStatus::SKIPPED) {} - - BenchmarkResult(const std::string& name) - : kernel_name(name), correctness_status_(CorrectnessStatus::SKIPPED) {} - - BenchmarkResult( - const std::string& kernel_name, - const std::string& operator_name) - : kernel_name(kernel_name), - operator_name(operator_name), - correctness_status_(CorrectnessStatus::SKIPPED) {} - - // Add timing for a single iteration - void add_iter_timing(float time_us); - - // Add per-shader timing for a single iteration - void add_shader_timing( - const std::string& shader_name, - float time_us, - const uint32_t gwg[3], - const uint32_t lwg[3]); - - // Get per-shader timing data - const std::vector& get_shader_timings() const { - return shader_timings_; - } - - // Getters - const std::string& get_kernel_name() const { - return kernel_name; - } - const std::string& get_operator_name() const { - return operator_name; - } - float get_avg_time_us() const; - size_t get_num_iterations() const { - return iter_timings.size(); - } - const std::vector& get_iter_timings() const { - return iter_timings; - } - CorrectnessStatus get_correctness_status() const { - return correctness_status_; - } - - // Setters - void set_kernel_name(const std::string& name) { - kernel_name = name; - } - void set_operator_name(const std::string& name) { - operator_name = name; - } - void set_correctness_status(CorrectnessStatus status) { - correctness_status_ = status; - } - - // Statistics - float get_min_time_us() const; - float get_max_time_us() const; - float get_std_dev_us() const; - - // Clear all timings - void clear_timings() { - iter_timings.clear(); - } - - // Print progress for this benchmark result - void print_summary( - int case_number, - const std::string& size_info, - float total_gflops) const; - - private: - std::string kernel_name; - std::string operator_name; - std::vector - iter_timings; // Individual iteration timings in microseconds - std::vector shader_timings_; // Per-shader timing data - CorrectnessStatus correctness_status_; -}; - -// Test result collection and processing -class TestResult { - public: - TestResult() : gflops_(0.0f), correctness_passed_(true) {} - TestResult(const std::string& operation_name) - : operation_name_(operation_name), - gflops_(0.0f), - correctness_passed_(true) {} - - // Add a benchmark result - void add_result(const BenchmarkResult& result); - void add_result(BenchmarkResult&& result); - - // Getters - const std::string& get_operation_name() const { - return operation_name_; - } - float get_gflops() const { - return gflops_; - } - bool get_correctness_passed() const { - return correctness_passed_; - } - size_t size() const { - return results_.size(); - } - bool empty() const { - return results_.empty(); - } - - // Setters - void set_gflops(float gflops_val) { - gflops_ = gflops_val; - } - void set_correctness_passed(bool passed) { - correctness_passed_ = passed; - } - - // Access results - const BenchmarkResult& operator[](size_t index) const { - return results_[index]; - } - BenchmarkResult& operator[](size_t index) { - return results_[index]; - } - const std::vector& get_results() const { - return results_; - } - - // Iterator support - std::vector::iterator begin() { - return results_.begin(); - } - std::vector::iterator end() { - return results_.end(); - } - std::vector::const_iterator begin() const { - return results_.begin(); - } - std::vector::const_iterator end() const { - return results_.end(); - } - - // Processing and analysis - void print_summary() const; - void print_detailed_results() const; - void print_statistics() const; - void print_brief_summary() const; - - // Get aggregate statistics - float get_total_avg_time_us() const; - float get_total_gflops() const; - size_t get_passed_count() const; - size_t get_failed_count() const; - - // Find best/worst performing results - const BenchmarkResult* get_fastest_result() const; - const BenchmarkResult* get_slowest_result() const; - const BenchmarkResult* get_highest_gflops_result() const; - - // Clear all results - void clear() { - results_.clear(); - } - - // Set operation name - void set_operation_name(const std::string& name) { - operation_name_ = name; - } - - private: - std::string operation_name_; - std::vector results_; - float gflops_; - bool correctness_passed_; -}; - -// -// Test case execution -// - -using FlopCalculatorFunc = std::function; - -// Default FLOP calculation function (assumes 1 FLOP per element) -int64_t default_flop_calculator(const TestCase& test_case); - -using ReferenceComputeFunc = std::function; - -// Half-open index range of the operator's own dispatch nodes within a -// benchmark graph's execute_nodes(). Staging upload nodes precede it, staging -// download nodes follow it. -struct OpNodeRange { - size_t begin = 0; - size_t end = 0; -}; - -// A benchmark graph plus the location of its repeatable operator nodes. The -// graph is heap-held: ComputeGraph owns its Context and must never be moved -// (a moved-from graph's destructor dereferences a null context). -struct BenchmarkGraph { - std::unique_ptr graph; - OpNodeRange op_nodes; -}; - -// Benchmark-only executor that records a graph's execute nodes into a single -// reusable command buffer, then replays it on every execute(). Staging uploads -// are encoded once, operator nodes N times, staging downloads once, so each -// iteration performs the same work as the old stacked-nodes layout (1 upload + -// N ops + 1 download) and the per-invocation divisor is unchanged. Production -// ComputeGraph::execute() behavior is unchanged. -// -// Notes for interpreting benchmark numbers: -// - Submit granularity differs from stacking N distinct nodes: all encodings -// live in one command buffer with one submit per iteration (the old path -// could split across command buffers at the node-count threshold), so -// per-dispatch times may shift systematically against older data. -// - Resize functions run once via propagate_resize() in execute_test_case -// before recording; replay itself never re-triggers resize. -// - Repeated encodings share one node/dispatch id, so per-repetition -// querypool attribution is unavailable (aggregation keys on kernel name). -class RepeatedGraphExecutor final { - public: - RepeatedGraphExecutor( - ComputeGraph& graph, - int repetitions, - OpNodeRange op_nodes); - void execute(); - - private: - ComputeGraph& graph_; - std::unique_ptr command_; -}; - -// Runs a measurement at the given chained_dispatches factor. The operator is -// built once, then its execute nodes are encoded that many times into a -// benchmark-only reusable command buffer. The probe-then-scale orchestration -// lives in execute_test_cases(). -// -// write_outputs controls whether the graph's staging output buffers are copied -// back into test_case.outputs() at the end of the run. The probe path needs -// write_outputs=true so the correctness check has a clean single-dispatch -// reference. The benchmarking path passes write_outputs=false to avoid the -// per-iter GPU->CPU copy cost. -BenchmarkResult execute_test_case( - TestCase& test_case, - int warmup_runs = 1, - int benchmark_runs = 1, - int chained_dispatches = 1, - bool write_outputs = true); - -TestResult execute_test_cases( - std::function()> test_case_generator, - FlopCalculatorFunc flop_calculator, - const std::string& operation_name = "Operation", - int warmup_runs = 1, - int benchmark_runs = 1, - ReferenceComputeFunc reference_compute_func = nullptr); - -TestResult execute_test_cases( - std::function()> test_case_generator, - const std::string& operation_name = "Operation", - int warmup_runs = 1, - int benchmark_runs = 1, - ReferenceComputeFunc reference_compute_func = nullptr); - -// -// Print utilities -// - -void print_performance_header(); -void print_separator(); - -void print_valuespec_data( - const ValueSpec& spec, - const std::string& name = "ValueSpec", - const bool print_ref_data = false, - size_t max_elements = 20, - int precision = 6); - -ValueRef quantized_weights_canvas( - ComputeGraph& graph, - const ValueRef weight_ref); - -ValueRef float_tensor_canvas(ComputeGraph& graph, const ValueRef weight_ref); - -// Compute weight sums for quantized operations (linear and convolution) -void compute_weight_sums( - ValueSpec& weight_sums, - const ValueSpec& quantized_weight, - int64_t out_features, - int64_t elements_per_output_feature); - -// Compute weight sums for 4D quantized conv2d operations -// Weight layout: [C_out, K_h, K_w, align_up_4(C_in_per_group)] -void compute_weight_sums_4d( - ValueSpec& weight_sums, - const ValueSpec& quantized_weight, - int64_t out_channels, - int64_t kernel_h, - int64_t kernel_w, - int64_t aligned_in_channels); - -// Compute weight sums for 4-bit group symmetric quantized weights -void compute_weight_sums_4bit_grouped( - ValueSpec& weight_sums, - const ValueSpec& quantized_weight, - int64_t num_groups, - int64_t out_features, - int64_t group_size); - -// Half-precision conversion utilities -uint16_t float_to_half(float value); -float half_to_float(uint16_t half_val); - -// Setup compute graph based on TestCase and operation name. The op function is -// invoked once. op_invocations_per_execute is used only to reserve enough -// descriptor capacity for benchmark-only repeated command encoding. Returns -// the graph plus the range of the operator's own nodes for repeated encoding. -BenchmarkGraph setup_compute_graph( - TestCase& test_case, - std::string op_name, - int op_invocations_per_execute = 1); - -} // namespace prototyping -} // namespace vulkan -} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/utils_test.cpp b/backends/vulkan/test/custom_ops/utils_test.cpp index 6d2de13d71e..104d585769e 100644 --- a/backends/vulkan/test/custom_ops/utils_test.cpp +++ b/backends/vulkan/test/custom_ops/utils_test.cpp @@ -6,7 +6,7 @@ #include -#include "utils.h" +#include namespace executorch::vulkan::prototyping { namespace { diff --git a/backends/vulkan/test/op_tests/conv2d_gemm_dynamic_test.cpp b/backends/vulkan/test/op_tests/conv2d_gemm_dynamic_test.cpp index f993c453fb0..7b4954113ea 100644 --- a/backends/vulkan/test/op_tests/conv2d_gemm_dynamic_test.cpp +++ b/backends/vulkan/test/op_tests/conv2d_gemm_dynamic_test.cpp @@ -35,12 +35,13 @@ using namespace vkcompute; // // The im2col + GEMM conv path (conv2d_gemm_impl) materializes its im2col matrix // in tiles of output-height rows bounded to a byte budget. The static op-test -// suite (test/custom_ops/test_conv2d.cpp) only validates a single shape per -// graph. This test exercises the path under an actual trigger_resize: it builds -// the graph at an upper-bound input shape that forces MULTIPLE tiles, then -// resizes the input across tile boundaries (so trailing tiles must no-op via -// the shader's `oh < H_out` guard and the scratch must track the smaller shape) -// and verifies the output against a reference at every shape. +// suite (test/ops/benchmark/cases/conv2d/test_conv2d.cpp) only validates a +// single shape per graph. This test exercises the path under an actual +// trigger_resize: it builds the graph at an upper-bound input shape that forces +// MULTIPLE tiles, then resizes the input across tile boundaries (so trailing +// tiles must no-op via the shader's `oh < H_out` guard and the scratch must +// track the smaller shape) and verifies the output against a reference at every +// shape. // // Build-time upper bound: storage and the fixed num_tiles are built at the // initial input shape, so every resized shape MUST be <= the initial shape per diff --git a/backends/vulkan/test/ops/benchmark/BUCK b/backends/vulkan/test/ops/benchmark/BUCK new file mode 100644 index 00000000000..f559a6f1cfe --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/BUCK @@ -0,0 +1,9 @@ +# Any targets that should be shared between fbcode and xplat must be defined in +# targets.bzl. + +load("@fbsource//tools/build_defs:fbsource_utils.bzl", "is_fbcode") +load(":targets.bzl", "define_common_targets") + +oncall("executorch") + +define_common_targets(is_fbcode = is_fbcode()) diff --git a/backends/vulkan/test/custom_ops/CMakeLists.txt b/backends/vulkan/test/ops/benchmark/CMakeLists.txt similarity index 63% rename from backends/vulkan/test/custom_ops/CMakeLists.txt rename to backends/vulkan/test/ops/benchmark/CMakeLists.txt index f29b518ec06..ff9138779ec 100644 --- a/backends/vulkan/test/custom_ops/CMakeLists.txt +++ b/backends/vulkan/test/ops/benchmark/CMakeLists.txt @@ -25,7 +25,7 @@ message(STATUS "VULKAN_CXX_FLAGS: ${VULKAN_CXX_FLAGS}") # Only build if Vulkan was compiled if(TARGET vulkan_backend) if(NOT EXECUTORCH_ROOT) - set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../..) + set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../../..) endif() if(NOT PYTHON_EXECUTABLE) @@ -47,23 +47,30 @@ if(TARGET vulkan_backend) ) # Prototyping utility files - set(PROTOTYPING_UTILS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}) + set(PROTOTYPING_UTILS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/framework) set(PROTOTYPING_UTILS_CPP - ${CMAKE_CURRENT_SOURCE_DIR}/utils.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/conv2d_utils.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/cm_utils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/cm_utils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/config.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/conv2d_utils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/results.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/runner.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/test_case.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/value_spec.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/weight_utils.cpp ) # Prototyping shaders message(STATUS "shader stuff") - set(PROTOTYPING_SHADERS_PATH ${CMAKE_CURRENT_SOURCE_DIR}/glsl) + set(PROTOTYPING_SHADERS_PATH ${CMAKE_CURRENT_SOURCE_DIR}/test_ops/glsl) gen_vulkan_shader_lib_cpp(${PROTOTYPING_SHADERS_PATH}) vulkan_shader_lib(prototyping_shaderlib ${generated_spv_cpp}) target_compile_options(prototyping_shaderlib PRIVATE ${VULKAN_CXX_FLAGS}) message(STATUS "done shader stuff") # Operator implementations library - file(GLOB OPERATOR_IMPL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/impl/*.cpp) + file(GLOB OPERATOR_IMPL_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/test_ops/impl/*.cpp + ) add_library(operator_implementations STATIC ${OPERATOR_IMPL_SOURCES}) target_include_directories( operator_implementations PRIVATE ${COMMON_INCLUDES} @@ -79,9 +86,9 @@ if(TARGET vulkan_backend) executorch_target_link_options_shared_lib(operator_implementations) # Function to create operator prototype binaries - function(add_operator_prototype OPERATOR_NAME) + function(add_operator_prototype OPERATOR_NAME SUBDIR) set(TARGET_NAME ${OPERATOR_NAME}) - set(SOURCE_FILE ${CMAKE_CURRENT_SOURCE_DIR}/${OPERATOR_NAME}.cpp) + set(SOURCE_FILE ${CMAKE_CURRENT_SOURCE_DIR}/${SUBDIR}/${OPERATOR_NAME}.cpp) add_executable(${TARGET_NAME} ${SOURCE_FILE} ${PROTOTYPING_UTILS_CPP}) target_include_directories(${TARGET_NAME} PRIVATE ${COMMON_INCLUDES}) @@ -94,18 +101,20 @@ if(TARGET vulkan_backend) endfunction() # Define operator prototypes - add_operator_prototype(test_add) - add_operator_prototype(test_q8csw_linear) - add_operator_prototype(test_q8csw_conv2d) - add_operator_prototype(test_q4gsw_linear) - add_operator_prototype(test_choose_qparams_per_row) - add_operator_prototype(test_q8ta_qdq) - add_operator_prototype(test_q8ta_clone) - add_operator_prototype(test_q8ta_binary) - add_operator_prototype(test_q8ta_conv2d) - add_operator_prototype(test_q8ta_conv2d_pw) - add_operator_prototype(test_q8ta_conv2d_dw) - add_operator_prototype(test_mm) - add_operator_prototype(test_coopmat_probe) - add_operator_prototype(test_coopmat_linear_bench) + add_operator_prototype(test_add cases/add) + add_operator_prototype(test_q8csw_linear cases/q8csw_linear) + add_operator_prototype(test_q8csw_conv2d cases/q8csw_conv2d) + add_operator_prototype(test_q4gsw_linear cases/q4gsw_linear) + add_operator_prototype( + test_choose_qparams_per_row cases/choose_qparams_per_row + ) + add_operator_prototype(test_q8ta_qdq cases/q8ta/qdq) + add_operator_prototype(test_q8ta_clone cases/q8ta/clone) + add_operator_prototype(test_q8ta_binary cases/q8ta/binary) + add_operator_prototype(test_q8ta_conv2d cases/q8ta/conv2d) + add_operator_prototype(test_q8ta_conv2d_pw cases/q8ta/conv2d) + add_operator_prototype(test_q8ta_conv2d_dw cases/q8ta/conv2d) + add_operator_prototype(test_mm cases/mm) + add_operator_prototype(test_coopmat_probe framework) + add_operator_prototype(test_coopmat_linear_bench cases/q4gsw_linear) endif() diff --git a/backends/vulkan/test/custom_ops/build_and_run.sh b/backends/vulkan/test/ops/benchmark/build_and_run.sh similarity index 89% rename from backends/vulkan/test/custom_ops/build_and_run.sh rename to backends/vulkan/test/ops/benchmark/build_and_run.sh index 2641970faf8..a03aa3032a3 100755 --- a/backends/vulkan/test/custom_ops/build_and_run.sh +++ b/backends/vulkan/test/ops/benchmark/build_and_run.sh @@ -49,28 +49,28 @@ configure_and_build_tests() { local android_args="" if [[ "$ANDROID_MODE" == "true" ]]; then - cmake backends/vulkan/test/custom_ops/ \ + cmake backends/vulkan/test/ops/benchmark/ \ -DCMAKE_INSTALL_PREFIX=$CMAKE_OUT_DIR \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \ -DANDROID_ABI=arm64-v8a \ -DANDROID_PLATFORM=android-28 \ -DGLSLC_PATH=$(which glslc) \ - -B$CMAKE_OUT_DIR/backends/vulkan/test/custom_ops + -B$CMAKE_OUT_DIR/backends/vulkan/test/ops/benchmark else - cmake backends/vulkan/test/custom_ops/ \ + cmake backends/vulkan/test/ops/benchmark/ \ -DCMAKE_INSTALL_PREFIX=$CMAKE_OUT_DIR \ -DCMAKE_BUILD_TYPE=Debug \ -DGLSLC_PATH=$(which glslc) \ - -B$CMAKE_OUT_DIR/backends/vulkan/test/custom_ops + -B$CMAKE_OUT_DIR/backends/vulkan/test/ops/benchmark fi - cmake --build $CMAKE_OUT_DIR/backends/vulkan/test/custom_ops -j$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu) + 1 )) --target all + cmake --build $CMAKE_OUT_DIR/backends/vulkan/test/ops/benchmark -j$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu) + 1 )) --target all } build_tests() { - cmake --build $CMAKE_OUT_DIR/backends/vulkan/test/custom_ops -j$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu) + 1 )) --target all + cmake --build $CMAKE_OUT_DIR/backends/vulkan/test/ops/benchmark -j$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu) + 1 )) --target all } # Function to rebuild both main and tests @@ -90,7 +90,7 @@ clean_and_rebuild() { execute_binary() { local binary_name="$1" if [[ -n "$binary_name" ]]; then - local binary_path="$CMAKE_OUT_DIR/backends/vulkan/test/custom_ops/$binary_name" + local binary_path="$CMAKE_OUT_DIR/backends/vulkan/test/ops/benchmark/$binary_name" echo "Executing binary: $binary_path" if [[ "$ANDROID_MODE" == "true" ]]; then diff --git a/backends/vulkan/test/custom_ops/test_add.cpp b/backends/vulkan/test/ops/benchmark/cases/add/test_add.cpp similarity index 98% rename from backends/vulkan/test/custom_ops/test_add.cpp rename to backends/vulkan/test/ops/benchmark/cases/add/test_add.cpp index bd7c644c2fd..fed59acedf0 100644 --- a/backends/vulkan/test/custom_ops/test_add.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/add/test_add.cpp @@ -6,9 +6,9 @@ #include #include +#include #include #include -#include "utils.h" using namespace executorch::vulkan::prototyping; diff --git a/backends/vulkan/test/custom_ops/test_choose_qparams_per_row.cpp b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/test_choose_qparams_per_row.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_choose_qparams_per_row.cpp rename to backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/test_choose_qparams_per_row.cpp index 69f7812e4ab..48139371d89 100644 --- a/backends/vulkan/test/custom_ops/test_choose_qparams_per_row.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/test_choose_qparams_per_row.cpp @@ -6,11 +6,11 @@ #include #include +#include #include #include #include #include -#include "utils.h" #include diff --git a/backends/vulkan/test/custom_ops/test_conv1d_dw.cpp b/backends/vulkan/test/ops/benchmark/cases/conv1d/test_conv1d_dw.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_conv1d_dw.cpp rename to backends/vulkan/test/ops/benchmark/cases/conv1d/test_conv1d_dw.cpp index 92539037f75..8e1234611c9 100644 --- a/backends/vulkan/test/custom_ops/test_conv1d_dw.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/conv1d/test_conv1d_dw.cpp @@ -10,7 +10,7 @@ #include #include -#include "utils.h" +#include using namespace executorch::vulkan::prototyping; using namespace vkcompute; diff --git a/backends/vulkan/test/custom_ops/test_conv1d_pw.cpp b/backends/vulkan/test/ops/benchmark/cases/conv1d/test_conv1d_pw.cpp similarity index 98% rename from backends/vulkan/test/custom_ops/test_conv1d_pw.cpp rename to backends/vulkan/test/ops/benchmark/cases/conv1d/test_conv1d_pw.cpp index dc40d1ee795..aab1862c75d 100644 --- a/backends/vulkan/test/custom_ops/test_conv1d_pw.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/conv1d/test_conv1d_pw.cpp @@ -10,7 +10,7 @@ #include #include -#include "utils.h" +#include using namespace executorch::vulkan::prototyping; using namespace vkcompute; diff --git a/backends/vulkan/test/custom_ops/test_conv2d.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_conv2d.cpp rename to backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d.cpp index 2f21f0532fe..83a604345b8 100644 --- a/backends/vulkan/test/custom_ops/test_conv2d.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d.cpp @@ -10,8 +10,8 @@ #include #include -#include "conv2d_utils.h" -#include "utils.h" +#include +#include using namespace executorch::vulkan::prototyping; using namespace vkcompute; diff --git a/backends/vulkan/test/custom_ops/test_conv2d_dw.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_dw.cpp similarity index 98% rename from backends/vulkan/test/custom_ops/test_conv2d_dw.cpp rename to backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_dw.cpp index e0e8a55b815..4ba7a3d6286 100644 --- a/backends/vulkan/test/custom_ops/test_conv2d_dw.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_dw.cpp @@ -10,8 +10,8 @@ #include #include -#include "conv2d_utils.h" -#include "utils.h" +#include +#include using namespace executorch::vulkan::prototyping; using namespace vkcompute; diff --git a/backends/vulkan/test/custom_ops/test_conv2d_pw.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_pw.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_conv2d_pw.cpp rename to backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_pw.cpp index 30cdb352248..da38a744417 100644 --- a/backends/vulkan/test/custom_ops/test_conv2d_pw.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_pw.cpp @@ -10,7 +10,7 @@ #include #include -#include "utils.h" +#include using namespace executorch::vulkan::prototyping; using namespace vkcompute; diff --git a/backends/vulkan/test/custom_ops/test_embedding_q4gsw.cpp b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_embedding_q4gsw.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_embedding_q4gsw.cpp rename to backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_embedding_q4gsw.cpp index 7f38b4de77b..20fcce3c5ca 100644 --- a/backends/vulkan/test/custom_ops/test_embedding_q4gsw.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_embedding_q4gsw.cpp @@ -10,7 +10,7 @@ #include #include -#include "utils.h" +#include using namespace executorch::vulkan::prototyping; diff --git a/backends/vulkan/test/custom_ops/test_mm.cpp b/backends/vulkan/test/ops/benchmark/cases/mm/test_mm.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_mm.cpp rename to backends/vulkan/test/ops/benchmark/cases/mm/test_mm.cpp index 4a8d9de49b6..608f0fa369a 100644 --- a/backends/vulkan/test/custom_ops/test_mm.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/mm/test_mm.cpp @@ -10,7 +10,7 @@ #include #include -#include "utils.h" +#include using namespace executorch::vulkan::prototyping; using namespace vkcompute; diff --git a/backends/vulkan/test/custom_ops/test_coopmat_linear_bench.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_coopmat_linear_bench.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_coopmat_linear_bench.cpp rename to backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_coopmat_linear_bench.cpp index d57d41266a5..8cd9342f15e 100644 --- a/backends/vulkan/test/custom_ops/test_coopmat_linear_bench.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_coopmat_linear_bench.cpp @@ -18,11 +18,11 @@ #include #include #include +#include #include #include #include #include -#include "utils.h" using namespace executorch::vulkan::prototyping; using namespace vkcompute; diff --git a/backends/vulkan/test/custom_ops/test_fpa_q4gsw_linear.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_fpa_q4gsw_linear.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_fpa_q4gsw_linear.cpp rename to backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_fpa_q4gsw_linear.cpp index 212ba03ddaa..ef0249d4267 100644 --- a/backends/vulkan/test/custom_ops/test_fpa_q4gsw_linear.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_fpa_q4gsw_linear.cpp @@ -44,9 +44,9 @@ #include #include +#include #include #include -#include "utils.h" using namespace executorch::vulkan::prototyping; diff --git a/backends/vulkan/test/custom_ops/test_q4gsw_linear.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_q4gsw_linear.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_q4gsw_linear.cpp rename to backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_q4gsw_linear.cpp index db188d21c30..cb06f9692e0 100644 --- a/backends/vulkan/test/custom_ops/test_q4gsw_linear.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_q4gsw_linear.cpp @@ -6,11 +6,11 @@ #include #include +#include #include #include #include #include -#include "utils.h" #include diff --git a/backends/vulkan/test/custom_ops/test_q8csw_conv2d.cpp b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/test_q8csw_conv2d.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_q8csw_conv2d.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/test_q8csw_conv2d.cpp index be7a4cab2b8..558f9228355 100644 --- a/backends/vulkan/test/custom_ops/test_q8csw_conv2d.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/test_q8csw_conv2d.cpp @@ -6,10 +6,10 @@ #include #include +#include +#include #include #include -#include "conv2d_utils.h" -#include "utils.h" #include diff --git a/backends/vulkan/test/custom_ops/test_q8csw_linear.cpp b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_q8csw_linear.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_q8csw_linear.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_q8csw_linear.cpp index 5e9154b5648..33c6947e167 100644 --- a/backends/vulkan/test/custom_ops/test_q8csw_linear.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_q8csw_linear.cpp @@ -6,9 +6,9 @@ #include #include +#include #include #include -#include "utils.h" #include diff --git a/backends/vulkan/test/custom_ops/test_q8ta_binary.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/test_q8ta_binary.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_q8ta_binary.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/binary/test_q8ta_binary.cpp index 4a2654b00db..47b65fa2f1e 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_binary.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/test_q8ta_binary.cpp @@ -6,12 +6,12 @@ #include #include +#include #include #include #include #include #include -#include "utils.h" using namespace executorch::vulkan::prototyping; diff --git a/backends/vulkan/test/custom_ops/test_q8ta_clone.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_q8ta_clone.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_q8ta_clone.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_q8ta_clone.cpp index 4936c3702cf..2f6ea11b6c8 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_clone.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_q8ta_clone.cpp @@ -6,11 +6,11 @@ #include #include +#include #include #include #include #include -#include "utils.h" #include diff --git a/backends/vulkan/test/custom_ops/test_q8ta_conv2d.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_q8ta_conv2d.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d.cpp index 17045225a2c..683dbf8506c 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_conv2d.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d.cpp @@ -18,8 +18,8 @@ #include -#include "conv2d_utils.h" -#include "utils.h" +#include +#include // #define DEBUG_MODE diff --git a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_dw.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d_dw.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_q8ta_conv2d_dw.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d_dw.cpp index c573b8b5d39..6ff15f68f81 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_dw.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d_dw.cpp @@ -12,8 +12,8 @@ #include -#include "conv2d_utils.h" -#include "utils.h" +#include +#include // #define DEBUG_MODE diff --git a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_pw.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d_pw.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_q8ta_conv2d_pw.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d_pw.cpp index 5a773f43319..8062b25d31d 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_pw.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d_pw.cpp @@ -14,8 +14,8 @@ #include -#include "conv2d_utils.h" -#include "utils.h" +#include +#include // #define DEBUG_MODE diff --git a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_transposed.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/test_q8ta_conv2d_transposed.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_q8ta_conv2d_transposed.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/test_q8ta_conv2d_transposed.cpp index b82cb19e614..4ead8123f45 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_transposed.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/test_q8ta_conv2d_transposed.cpp @@ -12,8 +12,8 @@ #include -#include "conv2d_utils.h" -#include "utils.h" +#include +#include // #define DEBUG_MODE diff --git a/backends/vulkan/test/custom_ops/test_q8ta_linear.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/test_q8ta_linear.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_q8ta_linear.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/linear/test_q8ta_linear.cpp index 2b65b53c896..a935508bc19 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_linear.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/test_q8ta_linear.cpp @@ -12,7 +12,7 @@ #include -#include "utils.h" +#include using namespace executorch::vulkan::prototyping; diff --git a/backends/vulkan/test/custom_ops/test_q8ta_pixel_shuffle.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/test_q8ta_pixel_shuffle.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_q8ta_pixel_shuffle.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/test_q8ta_pixel_shuffle.cpp index b932d1e92d3..948ee347520 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_pixel_shuffle.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/test_q8ta_pixel_shuffle.cpp @@ -6,12 +6,12 @@ #include #include +#include #include #include #include #include #include -#include "utils.h" #include diff --git a/backends/vulkan/test/custom_ops/test_q8ta_qdq.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_q8ta_qdq.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_q8ta_qdq.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_q8ta_qdq.cpp index af48018d31a..d11988dd192 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_qdq.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_q8ta_qdq.cpp @@ -6,11 +6,11 @@ #include #include +#include #include #include #include #include -#include "utils.h" #include diff --git a/backends/vulkan/test/custom_ops/test_q8ta_unary.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/test_q8ta_unary.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_q8ta_unary.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/unary/test_q8ta_unary.cpp index a4072c01f02..43eff9bc8ee 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_unary.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/test_q8ta_unary.cpp @@ -6,11 +6,11 @@ #include #include +#include #include #include #include #include -#include "utils.h" #include diff --git a/backends/vulkan/test/custom_ops/test_sdpa.cpp b/backends/vulkan/test/ops/benchmark/cases/sdpa/test_sdpa.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/test_sdpa.cpp rename to backends/vulkan/test/ops/benchmark/cases/sdpa/test_sdpa.cpp index 55d2173e3cd..d71860c5178 100644 --- a/backends/vulkan/test/custom_ops/test_sdpa.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/sdpa/test_sdpa.cpp @@ -13,7 +13,7 @@ #include #include -#include "utils.h" +#include using namespace executorch::vulkan::prototyping; using namespace vkcompute; diff --git a/backends/vulkan/test/custom_ops/cm_utils.cpp b/backends/vulkan/test/ops/benchmark/framework/cm_utils.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/cm_utils.cpp rename to backends/vulkan/test/ops/benchmark/framework/cm_utils.cpp diff --git a/backends/vulkan/test/custom_ops/cm_utils.h b/backends/vulkan/test/ops/benchmark/framework/cm_utils.h similarity index 100% rename from backends/vulkan/test/custom_ops/cm_utils.h rename to backends/vulkan/test/ops/benchmark/framework/cm_utils.h diff --git a/backends/vulkan/test/ops/benchmark/framework/config.cpp b/backends/vulkan/test/ops/benchmark/framework/config.cpp new file mode 100644 index 00000000000..ada88cb840b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/config.cpp @@ -0,0 +1,55 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "config.h" + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// Output and latency printing utilities +namespace { +static int print_output_enabled = 0; +static int print_latencies_enabled = 0; +static int gpu_timestamps_enabled = 0; +static int debugging_enabled = 0; +} // namespace + +bool print_output() { + return print_output_enabled > 0; +} + +void set_print_output(bool print_output) { + print_output_enabled = print_output ? 1 : 0; +} + +bool print_latencies() { + return print_latencies_enabled > 0; +} + +void set_print_latencies(bool print_latencies) { + print_latencies_enabled = print_latencies ? 1 : 0; +} + +bool use_gpu_timestamps() { + return gpu_timestamps_enabled > 0; +} + +void set_use_gpu_timestamps(bool use_timestamps) { + gpu_timestamps_enabled = use_timestamps ? 1 : 0; +} + +bool debugging() { + return debugging_enabled > 0; +} + +void set_debugging(bool enable_debugging) { + debugging_enabled = enable_debugging ? 1 : 0; +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/config.h b/backends/vulkan/test/ops/benchmark/framework/config.h new file mode 100644 index 00000000000..6b2a3475234 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/config.h @@ -0,0 +1,31 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// +// Global configuration options +// + +bool print_output(); +void set_print_output(bool print_output); + +bool print_latencies(); +void set_print_latencies(bool print_latencies); + +bool use_gpu_timestamps(); +void set_use_gpu_timestamps(bool use_timestamps); + +bool debugging(); +void set_debugging(bool enable_debugging); + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/conv2d_utils.cpp b/backends/vulkan/test/ops/benchmark/framework/conv2d_utils.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/conv2d_utils.cpp rename to backends/vulkan/test/ops/benchmark/framework/conv2d_utils.cpp diff --git a/backends/vulkan/test/custom_ops/conv2d_utils.h b/backends/vulkan/test/ops/benchmark/framework/conv2d_utils.h similarity index 100% rename from backends/vulkan/test/custom_ops/conv2d_utils.h rename to backends/vulkan/test/ops/benchmark/framework/conv2d_utils.h diff --git a/backends/vulkan/test/ops/benchmark/framework/labels.h b/backends/vulkan/test/ops/benchmark/framework/labels.h new file mode 100644 index 00000000000..bcae57ccef8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/labels.h @@ -0,0 +1,136 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// NOLINTNEXTLINE(google-build-using-namespace) +using namespace vkcompute; + +// +// String utilities +// + +// Helper function to get abbreviated layout names for test case naming +inline std::string layout_abbrev(utils::GPUMemoryLayout layout) { + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (layout) { + case utils::kWidthPacked: + return "WP"; + case utils::kChannelsPacked: + return "CP"; + case utils::kPackedInt8_4W: + return "4W"; + case utils::kPackedInt8_4C: + return "4C"; + case utils::kPackedInt8_4W4C: + return "4W4C"; + case utils::kPackedInt8_4H4W: + return "4H4W"; + case utils::kPackedInt8_4C1W: + return "4C1W"; + default: + return "UNK"; + } +} + +// Helper function to get abbreviated storage type names for test case naming +inline std::string storage_type_abbrev(utils::StorageType storage_type) { + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (storage_type) { + case utils::kTexture3D: + return "Tex"; + case utils::kBuffer: + return "Buf"; + default: + return "UNK"; + } +} + +// Helper function to get combined storage type and layout representation +// Example: (kBuffer, kPackedInt8_4W4C) -> "Buf_4W4C" +inline std::string repr_str( + utils::StorageType storage_type, + utils::GPUMemoryLayout layout) { + return storage_type_abbrev(storage_type) + "(" + layout_abbrev(layout) + ")"; +} + +// Helper function to generate comma-separated shape string for test case naming +// Example: {1, 128, 56, 56} -> "1,128,56,56" +inline std::string shape_string(const std::vector& shape) { + std::string result; + for (size_t i = 0; i < shape.size(); ++i) { + if (i > 0) { + result += ","; + } + result += std::to_string(shape[i]); + } + return result; +} + +// Helper function to generate a bracketed shape string for test case naming +// Example: {1, 128, 56, 56} -> "[1,128,56,56]" +inline std::string shape_bracket(const std::vector& shape) { + return "[" + shape_string(shape) + "]"; +} + +// Short dtype symbol used in standardized test case labels +// Example: kFloat -> "f32", kHalf -> "f16", kChar -> "i8", kInt -> "i32" +inline std::string dtype_short(vkapi::ScalarType dtype) { + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + return "f32"; + case vkapi::kHalf: + return "f16"; + case vkapi::kChar: + return "i8"; + case vkapi::kByte: + return "u8"; + case vkapi::kInt: + return "i32"; + case vkapi::kBool: + return "b"; + default: + return "?"; + } +} + +// Build a standardized test label of the form: +// " [ ]" +// where is "->" when the two differ, or just +// "" when they match. Sections are separated by two spaces. If +// suffix is non-empty it is appended after a single space (allowing callers +// to pass e.g. "[general]" or "[gemv] +bias"). +inline std::string make_test_label( + const std::string& prefix, + const std::string& in_dtype, + const std::string& out_dtype, + const std::string& shape_str, + const std::string& storage_str, + const std::string& suffix = "") { + const std::string dtype_str = + (in_dtype == out_dtype) ? in_dtype : in_dtype + "->" + out_dtype; + std::string label = + prefix + " " + dtype_str + " " + shape_str + " " + storage_str; + if (!suffix.empty()) { + label += " " + suffix; + } + return label; +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/results.cpp b/backends/vulkan/test/ops/benchmark/framework/results.cpp new file mode 100644 index 00000000000..3a71c50237b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/results.cpp @@ -0,0 +1,468 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "results.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// BenchmarkResult implementation +void BenchmarkResult::add_iter_timing(float time_us) { + iter_timings.push_back(time_us); +} + +void BenchmarkResult::add_shader_timing( + const std::string& shader_name, + float time_us, + const uint32_t gwg[3], + const uint32_t lwg[3]) { + // Find existing shader timing or create new one + for (auto& st : shader_timings_) { + if (st.shader_name == shader_name) { + st.iter_timings_us.push_back(time_us); + // Work group sizes should be consistent across iterations + return; + } + } + // Not found, create new entry + ShaderTiming new_timing; + new_timing.shader_name = shader_name; + new_timing.iter_timings_us.push_back(time_us); + new_timing.gwg[0] = gwg[0]; + new_timing.gwg[1] = gwg[1]; + new_timing.gwg[2] = gwg[2]; + new_timing.lwg[0] = lwg[0]; + new_timing.lwg[1] = lwg[1]; + new_timing.lwg[2] = lwg[2]; + shader_timings_.push_back(std::move(new_timing)); +} + +float BenchmarkResult::get_avg_time_us() const { + if (iter_timings.empty()) { + return 0.0f; + } + + float sum = 0.0f; + for (float timing : iter_timings) { + sum += timing; + } + return sum / iter_timings.size(); +} + +float BenchmarkResult::get_min_time_us() const { + if (iter_timings.empty()) { + return 0.0f; + } + + return *std::min_element(iter_timings.begin(), iter_timings.end()); +} + +float BenchmarkResult::get_max_time_us() const { + if (iter_timings.empty()) { + return 0.0f; + } + + return *std::max_element(iter_timings.begin(), iter_timings.end()); +} + +float BenchmarkResult::get_std_dev_us() const { + if (iter_timings.size() <= 1) { + return 0.0f; + } + + float mean = get_avg_time_us(); + float sum_sq_diff = 0.0f; + + for (float timing : iter_timings) { + float diff = timing - mean; + sum_sq_diff += diff * diff; + } + + return std::sqrt(sum_sq_diff / (iter_timings.size() - 1)); +} + +void BenchmarkResult::print_summary( + int case_number, + const std::string& size_info, + float total_gflops) const { + (void)case_number; + static constexpr int OPERATOR_NAME_WIDTH = 50; + static constexpr int GLOBAL_WG_WIDTH = 16; + static constexpr int LOCAL_WG_WIDTH = 12; + static constexpr int KERNEL_NAME_WIDTH = 80; + static constexpr int SIZE_INFO_WIDTH = 20; + static constexpr int TIMING_WIDTH = 16; + static constexpr int GFLOPS_WIDTH = 14; + static constexpr int CORRECTNESS_WIDTH = 8; + + // Helper to truncate shader names longer than 46 chars to 44 chars + ".." + auto truncate_shader_name = [](const std::string& name) -> std::string { + if (name.length() > 46) { + return name.substr(0, 44) + ".."; + } + return name; + }; + + // Helper to format work group size as (x,y,z) + auto format_wg_size = [](const uint32_t wg[3]) -> std::string { + return "(" + std::to_string(wg[0]) + "," + std::to_string(wg[1]) + "," + + std::to_string(wg[2]) + ")"; + }; + + std::string correctness_str; + switch (correctness_status_) { + case CorrectnessStatus::SKIPPED: + correctness_str = "SKIPPED"; + break; + case CorrectnessStatus::PASSED: + correctness_str = "PASSED"; + break; + case CorrectnessStatus::FAILED: + correctness_str = "FAILED"; + break; + } + + // If we have per-shader timing data, print one line per shader plus overall + if (!shader_timings_.empty()) { + // If only one shader, print a single combined row + if (shader_timings_.size() == 1) { + const auto& st = shader_timings_[0]; + std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) + << truncate_shader_name(st.shader_name) << " " << std::left + << std::setw(GLOBAL_WG_WIDTH) << format_wg_size(st.gwg) + << std::left << std::setw(LOCAL_WG_WIDTH) + << format_wg_size(st.lwg) << std::left + << std::setw(KERNEL_NAME_WIDTH) << get_kernel_name() + << std::right << " " << std::setw(SIZE_INFO_WIDTH) << size_info + << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) + << get_avg_time_us() << " μs " << std::setw(GFLOPS_WIDTH) + << std::fixed << std::setprecision(3) << total_gflops + << " GFLOP/s " << std::setw(CORRECTNESS_WIDTH) + << correctness_str << std::endl; + } else { + // Multiple shaders: print individual shader lines (without GFLOP/s) + for (size_t i = 0; i < shader_timings_.size(); ++i) { + const auto& st = shader_timings_[i]; + float shader_avg_time = st.get_avg_time_us(); + + // Shader lines don't show test case info + std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) + << truncate_shader_name(st.shader_name) << " " << std::left + << std::setw(GLOBAL_WG_WIDTH) << format_wg_size(st.gwg) + << std::left << std::setw(LOCAL_WG_WIDTH) + << format_wg_size(st.lwg) << std::left + << std::setw(KERNEL_NAME_WIDTH) << "" << std::right << " " + << std::setw(SIZE_INFO_WIDTH) << "" << std::setw(TIMING_WIDTH) + << std::fixed << std::setprecision(3) << shader_avg_time + << " μs " << std::setw(GFLOPS_WIDTH) << "" << " " + << std::setw(CORRECTNESS_WIDTH) << "" << std::endl; + } + + // Print overall row with operator name, test case info, total time, and + // GFLOP/s + std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) + << get_operator_name() << " " << std::left + << std::setw(GLOBAL_WG_WIDTH) << "" << std::left + << std::setw(LOCAL_WG_WIDTH) << "" << std::left + << std::setw(KERNEL_NAME_WIDTH) << get_kernel_name() + << std::right << " " << std::setw(SIZE_INFO_WIDTH) << size_info + << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) + << get_avg_time_us() << " μs " << std::setw(GFLOPS_WIDTH) + << std::fixed << std::setprecision(3) << total_gflops + << " GFLOP/s " << std::setw(CORRECTNESS_WIDTH) + << correctness_str << std::endl; + } + + // Print separator line between test cases + } else { + // No per-shader timing data, use the original format + std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) + << get_operator_name() << " " << std::left + << std::setw(GLOBAL_WG_WIDTH) << "" << std::left + << std::setw(LOCAL_WG_WIDTH) << "" << std::left + << std::setw(KERNEL_NAME_WIDTH) << get_kernel_name() << std::right + << " " << std::setw(SIZE_INFO_WIDTH) << size_info + << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) + << get_avg_time_us() << " μs " << std::setw(GFLOPS_WIDTH) + << std::fixed << std::setprecision(3) << total_gflops + << " GFLOP/s " << std::setw(CORRECTNESS_WIDTH) << correctness_str + << std::endl; + } +} + +// TestResult implementation +void TestResult::add_result(const BenchmarkResult& result) { + results_.push_back(result); +} + +void TestResult::add_result(BenchmarkResult&& result) { + results_.push_back(std::move(result)); +} + +void TestResult::print_summary() const { + static constexpr int CASE_WIDTH = 100; + static constexpr int KERNEL_NAME_WIDTH = 20; + static constexpr int TIMING_WIDTH = 12; + static constexpr int PASS_WIDTH = 8; + + if (results_.empty()) { + std::cout << "No results to display" << std::endl; + return; + } + + std::cout << "\n=== " << operation_name_ + << " Performance Summary ===" << std::endl; + print_separator(); + + std::cout << std::left << std::setw(CASE_WIDTH) << "Case" << std::left + << std::setw(KERNEL_NAME_WIDTH) << "Kernel Name" << std::left + << std::setw(TIMING_WIDTH) << "Avg (μs)" << std::left + << std::setw(TIMING_WIDTH) << "Min (μs)" << std::left + << std::setw(TIMING_WIDTH) << "Max (μs)" << std::left + << std::setw(TIMING_WIDTH) << "Std Dev" << std::left + << std::setw(PASS_WIDTH) << "Pass" << std::endl; + print_separator(); + + for (size_t i = 0; i < results_.size(); ++i) { + const auto& result = results_[i]; + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + std::cout << std::left << std::setw(CASE_WIDTH) << i + 1 << std::left + << std::setw(KERNEL_NAME_WIDTH) + << result.get_kernel_name().substr(0, KERNEL_NAME_WIDTH - 1) + << std::left << std::setw(TIMING_WIDTH) << std::fixed + << std::setprecision(3) << result.get_avg_time_us() << std::left + << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) + << result.get_min_time_us() << std::left + << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) + << result.get_max_time_us() << std::left + << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) + << result.get_std_dev_us() << std::left << std::setw(PASS_WIDTH) + << (vulkan_execute_succeeded ? "✓" : "✗") << std::endl; + } + + print_separator(); + std::cout << "Total cases: " << results_.size() + << ", Passed: " << get_passed_count() + << ", Failed: " << get_failed_count() << std::endl; + std::cout << "Overall GFLOP/s: " << std::fixed << std::setprecision(3) + << gflops_ << std::endl; + std::cout << "Overall correctness: " + << (correctness_passed_ ? "PASSED" : "FAILED") << std::endl; +} + +void TestResult::print_detailed_results() const { + if (results_.empty()) { + std::cout << "No results to display" << std::endl; + return; + } + + std::cout << "\n=== " << operation_name_ + << " Detailed Results ===" << std::endl; + + for (size_t i = 0; i < results_.size(); ++i) { + const auto& result = results_[i]; + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + std::cout << "\nCase " << i + 1 << ": " << result.get_kernel_name() + << std::endl; + std::cout << " Iterations: " << result.get_num_iterations() << std::endl; + std::cout << " Average: " << std::fixed << std::setprecision(3) + << result.get_avg_time_us() << " μs" << std::endl; + std::cout << " Min: " << std::fixed << std::setprecision(3) + << result.get_min_time_us() << " μs" << std::endl; + std::cout << " Max: " << std::fixed << std::setprecision(3) + << result.get_max_time_us() << " μs" << std::endl; + std::cout << " Std Dev: " << std::fixed << std::setprecision(3) + << result.get_std_dev_us() << " μs" << std::endl; + std::cout << " Correctness: " + << (vulkan_execute_succeeded ? "PASSED" : "FAILED") << std::endl; + + if (result.get_num_iterations() > 0) { + std::cout << " Individual timings (μs): "; + const auto& timings = result.get_iter_timings(); + for (size_t j = 0; j < std::min(size_t(10), timings.size()); ++j) { + std::cout << std::fixed << std::setprecision(1) << timings[j]; + if (j < std::min(size_t(10), timings.size()) - 1) { + std::cout << ", "; + } + } + if (timings.size() > 10) { + std::cout << " ... (" << (timings.size() - 10) << " more)"; + } + std::cout << std::endl; + } + } + + std::cout << "\nOverall Results:" << std::endl; + std::cout << " Total GFLOP/s: " << std::fixed << std::setprecision(3) + << gflops_ << std::endl; + std::cout << " Overall correctness: " + << (correctness_passed_ ? "PASSED" : "FAILED") << std::endl; +} + +void TestResult::print_statistics() const { + if (results_.empty()) { + std::cout << "No results to display statistics for" << std::endl; + return; + } + + std::cout << "\n=== " << operation_name_ << " Statistics ===" << std::endl; + std::cout << "Total test cases: " << results_.size() << std::endl; + std::cout << "Passed: " << get_passed_count() << std::endl; + std::cout << "Failed: " << get_failed_count() << std::endl; + std::cout << "Success rate: " << std::fixed << std::setprecision(1) + << (100.0f * get_passed_count() / results_.size()) << "%" + << std::endl; + + if (get_passed_count() > 0) { + std::cout << "Total average time: " << std::fixed << std::setprecision(3) + << get_total_avg_time_us() << " μs" << std::endl; + std::cout << "Total GFLOP/s: " << std::fixed << std::setprecision(3) + << get_total_gflops() << std::endl; + + const auto* fastest = get_fastest_result(); + const auto* slowest = get_slowest_result(); + const auto* highest_gflops = get_highest_gflops_result(); + + if (fastest) { + std::cout << "Fastest case: " << fastest->get_kernel_name() << " (" + << std::fixed << std::setprecision(3) + << fastest->get_avg_time_us() << " μs)" << std::endl; + } + + if (slowest) { + std::cout << "Slowest case: " << slowest->get_kernel_name() << " (" + << std::fixed << std::setprecision(3) + << slowest->get_avg_time_us() << " μs)" << std::endl; + } + + if (highest_gflops) { + std::cout << "Best performing case: " << highest_gflops->get_kernel_name() + << " (" << std::fixed << std::setprecision(3) + << highest_gflops->get_avg_time_us() << " μs)" << std::endl; + } + } +} + +void TestResult::print_brief_summary() const { + print_separator(); + std::cout << "Summary Statistics:" << std::endl; + + if (get_passed_count() > 0) { + std::cout << "Average execution time: " << std::fixed + << std::setprecision(3) << get_total_avg_time_us() << " μs" + << std::endl; + std::cout << "Total throughput: " << std::fixed << std::setprecision(3) + << get_gflops() << " GFLOP/s" << std::endl; + std::cout << "Successful test cases: " << get_passed_count() << "/" + << size() << std::endl; + std::cout << "Overall correctness: " + << (get_correctness_passed() ? "PASSED" : "FAILED") << std::endl; + } else { + std::cout << "No successful test cases to report" << std::endl; + } +} + +float TestResult::get_total_avg_time_us() const { + if (results_.empty()) { + return 0.0f; + } + + float sum = 0.0f; + size_t count = 0; + + for (const auto& result : results_) { + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + if (vulkan_execute_succeeded) { + sum += result.get_avg_time_us(); + count++; + } + } + + return count > 0 ? sum / count : 0.0f; +} + +float TestResult::get_total_gflops() const { + return gflops_; +} + +size_t TestResult::get_passed_count() const { + size_t count = 0; + for (const auto& result : results_) { + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + if (vulkan_execute_succeeded) { + count++; + } + } + return count; +} + +size_t TestResult::get_failed_count() const { + return results_.size() - get_passed_count(); +} + +const BenchmarkResult* TestResult::get_fastest_result() const { + const BenchmarkResult* fastest = nullptr; + + for (const auto& result : results_) { + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + if (vulkan_execute_succeeded) { + if (!fastest || result.get_avg_time_us() < fastest->get_avg_time_us()) { + fastest = &result; + } + } + } + + return fastest; +} + +const BenchmarkResult* TestResult::get_slowest_result() const { + const BenchmarkResult* slowest = nullptr; + + for (const auto& result : results_) { + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + if (vulkan_execute_succeeded) { + if (!slowest || result.get_avg_time_us() > slowest->get_avg_time_us()) { + slowest = &result; + } + } + } + + return slowest; +} + +const BenchmarkResult* TestResult::get_highest_gflops_result() const { + // Since GFLOPS is now a TestResult-level metric rather than per-case, + // this method returns the fastest result as a proxy for highest performance + return get_fastest_result(); +} + +// Utility functions for printing +void print_performance_header() { + std::cout << "\n=== Compute Shader Performance Benchmark ===" << std::endl; +} + +void print_separator() { + std::cout << std::string(70, '-') << std::endl; +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/results.h b/backends/vulkan/test/ops/benchmark/framework/results.h new file mode 100644 index 00000000000..e957ddf8bce --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/results.h @@ -0,0 +1,235 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// +// BenchmarkResult +// + +enum class CorrectnessStatus { + SKIPPED, // No reference function provided + PASSED, // Reference function provided and validation passed + FAILED // Reference function provided but validation failed +}; + +// Per-shader timing data for detailed reporting +struct ShaderTiming { + std::string shader_name; + std::vector iter_timings_us; // Individual iteration timings + uint32_t gwg[3] = {0, 0, 0}; + uint32_t lwg[3] = {0, 0, 0}; + + float get_avg_time_us() const { + if (iter_timings_us.empty()) { + return 0.0f; + } + float sum = 0.0f; + for (float t : iter_timings_us) { + sum += t; + } + return sum / iter_timings_us.size(); + } +}; + +class BenchmarkResult { + public: + BenchmarkResult() : correctness_status_(CorrectnessStatus::SKIPPED) {} + + explicit BenchmarkResult(const std::string& name) + : kernel_name(name), correctness_status_(CorrectnessStatus::SKIPPED) {} + + BenchmarkResult( + const std::string& kernel_name, + const std::string& operator_name) + : kernel_name(kernel_name), + operator_name(operator_name), + correctness_status_(CorrectnessStatus::SKIPPED) {} + + // Add timing for a single iteration + void add_iter_timing(float time_us); + + // Add per-shader timing for a single iteration + void add_shader_timing( + const std::string& shader_name, + float time_us, + const uint32_t gwg[3], + const uint32_t lwg[3]); + + // Get per-shader timing data + const std::vector& get_shader_timings() const { + return shader_timings_; + } + + // Getters + const std::string& get_kernel_name() const { + return kernel_name; + } + const std::string& get_operator_name() const { + return operator_name; + } + float get_avg_time_us() const; + size_t get_num_iterations() const { + return iter_timings.size(); + } + const std::vector& get_iter_timings() const { + return iter_timings; + } + CorrectnessStatus get_correctness_status() const { + return correctness_status_; + } + + // Setters + void set_kernel_name(const std::string& name) { + kernel_name = name; + } + void set_operator_name(const std::string& name) { + operator_name = name; + } + void set_correctness_status(CorrectnessStatus status) { + correctness_status_ = status; + } + + // Statistics + float get_min_time_us() const; + float get_max_time_us() const; + float get_std_dev_us() const; + + // Clear all timings + void clear_timings() { + iter_timings.clear(); + } + + // Print progress for this benchmark result + void print_summary( + int case_number, + const std::string& size_info, + float total_gflops) const; + + private: + std::string kernel_name; + std::string operator_name; + std::vector + iter_timings; // Individual iteration timings in microseconds + std::vector shader_timings_; // Per-shader timing data + CorrectnessStatus correctness_status_; +}; + +// Test result collection and processing +class TestResult { + public: + TestResult() : gflops_(0.0f), correctness_passed_(true) {} + explicit TestResult(const std::string& operation_name) + : operation_name_(operation_name), + gflops_(0.0f), + correctness_passed_(true) {} + + // Add a benchmark result + void add_result(const BenchmarkResult& result); + void add_result(BenchmarkResult&& result); + + // Getters + const std::string& get_operation_name() const { + return operation_name_; + } + float get_gflops() const { + return gflops_; + } + bool get_correctness_passed() const { + return correctness_passed_; + } + size_t size() const { + return results_.size(); + } + bool empty() const { + return results_.empty(); + } + + // Setters + void set_gflops(float gflops_val) { + gflops_ = gflops_val; + } + void set_correctness_passed(bool passed) { + correctness_passed_ = passed; + } + + // Access results + const BenchmarkResult& operator[](size_t index) const { + return results_[index]; + } + BenchmarkResult& operator[](size_t index) { + return results_[index]; + } + const std::vector& get_results() const { + return results_; + } + + // Iterator support + std::vector::iterator begin() { + return results_.begin(); + } + std::vector::iterator end() { + return results_.end(); + } + std::vector::const_iterator begin() const { + return results_.begin(); + } + std::vector::const_iterator end() const { + return results_.end(); + } + + // Processing and analysis + void print_summary() const; + void print_detailed_results() const; + void print_statistics() const; + void print_brief_summary() const; + + // Get aggregate statistics + float get_total_avg_time_us() const; + float get_total_gflops() const; + size_t get_passed_count() const; + size_t get_failed_count() const; + + // Find best/worst performing results + const BenchmarkResult* get_fastest_result() const; + const BenchmarkResult* get_slowest_result() const; + const BenchmarkResult* get_highest_gflops_result() const; + + // Clear all results + void clear() { + results_.clear(); + } + + // Set operation name + void set_operation_name(const std::string& name) { + operation_name_ = name; + } + + private: + std::string operation_name_; + std::vector results_; + float gflops_; + bool correctness_passed_; +}; + +// +// Print utilities +// + +void print_performance_header(); +void print_separator(); + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/runner.cpp b/backends/vulkan/test/ops/benchmark/framework/runner.cpp new file mode 100644 index 00000000000..3d5ce763bb5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/runner.cpp @@ -0,0 +1,760 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "runner.h" +#include "config.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +namespace { + +// Upper bound on the auto-sized chained_dispatches factor. Caps the factor +// when an op is fast enough that target_us / probe_us would otherwise +// exceed this; past this point, stacking more dispatches doesn't materially +// improve measurement precision. +constexpr int kMaxChainedDispatches = 200; + +// Floor for probe_us to avoid division-by-zero / a runaway chained_dispatches +// factor when an op finishes faster than the wall clock can resolve. +constexpr float kMinProbeTimeUs = 1.0f; + +} // namespace + +// Benchmark graphs are immutable after input upload, so the operator nodes can +// be recorded repeatedly. Staging uploads/downloads are encoded once: +// repeating them would redo host-visible copies on every repetition and bias +// per-invocation timings. +RepeatedGraphExecutor::RepeatedGraphExecutor( + ComputeGraph& graph, + int repetitions, + OpNodeRange op_nodes) + : graph_(graph) { + VK_CHECK_COND(repetitions > 0); + VK_CHECK_COND(op_nodes.begin <= op_nodes.end); + VK_CHECK_COND(op_nodes.end <= graph_.execute_nodes().size()); + + api::Context* const context = graph_.context(); + context->flush(); + context->set_cmd(/*reusable=*/true); + context->cmd_reset_querypool(); + + for (size_t i = 0; i < op_nodes.begin; ++i) { + graph_.execute_nodes()[i]->encode(&graph_); + } + for (int i = 0; i < repetitions; ++i) { + for (size_t j = op_nodes.begin; j < op_nodes.end; ++j) { + graph_.execute_nodes()[j]->encode(&graph_); + } + } + for (size_t i = op_nodes.end; i < graph_.execute_nodes().size(); ++i) { + graph_.execute_nodes()[i]->encode(&graph_); + } + + command_ = + std::make_unique(std::move(context->extract_cmd())); +} + +void RepeatedGraphExecutor::execute() { + api::Context* const context = graph_.context(); + command_->end(); + + // Intentionally bypasses Context::submit_cmd_to_gpu(): its submit-count + // bookkeeping and threshold-split path only serve submit_compute_job + // recording, which benchmarks don't use. + vkapi::VulkanFence fence = context->fences().get_fence(); + context->adapter_ptr()->submit_cmd( + context->queue(), + command_->get_submit_handle(/*final_use=*/false), + fence.get_submit_handle()); + fence.wait(); + context->fences().return_fence(fence); +} + +// Helper function to collect GPU timing from querypool +float collect_gpu_timing_us( + ComputeGraph& graph, + const std::vector& shader_filter) { + graph.context()->querypool().extract_results(); + const auto results = graph.context()->querypool().get_shader_timestamp_data(); + if (!results.empty()) { + // Sum durations of all shaders that don't match any pattern in + // shader_filter + float total_duration_us = 0.0f; + for (const auto& shader_result : results) { + bool filtered = false; + // Check if this shader matches any filter pattern + for (const auto& filter_pattern : shader_filter) { + if (shader_result.kernel_name.find(filter_pattern) != + std::string::npos) { + filtered = true; + break; + } + } + + if (!filtered) { + // Calculate duration from start and end times, convert from ns to μs + uint64_t duration_ns = + shader_result.end_time_ns - shader_result.start_time_ns; + total_duration_us += static_cast(duration_ns) / 1000.0f; + } + } + return total_duration_us; + } + return 0.0f; +} + +// Helper function to collect per-shader GPU timing from querypool +// Returns a map of shader_name -> timing_us for non-filtered shaders +std::unordered_map collect_per_shader_timing_us( + ComputeGraph& graph, + const std::vector& shader_filter) { + std::unordered_map shader_timings; + + graph.context()->querypool().extract_results(); + const auto results = graph.context()->querypool().get_shader_timestamp_data(); + for (const auto& shader_result : results) { + bool filtered = false; + // Check if this shader matches any filter pattern + for (const auto& filter_pattern : shader_filter) { + if (shader_result.kernel_name.find(filter_pattern) != std::string::npos) { + filtered = true; + break; + } + } + + if (!filtered) { + // Calculate duration from start and end times, convert from ns to μs + uint64_t duration_ns = + shader_result.end_time_ns - shader_result.start_time_ns; + float duration_us = static_cast(duration_ns) / 1000.0f; + // Accumulate timing for shaders with the same name + shader_timings[shader_result.kernel_name] += duration_us; + } + } + return shader_timings; +} + +// Default FLOP calculation function (assumes 1 FLOP per element) +int64_t default_flop_calculator(const TestCase& test_case) { + // Calculate total elements from the first input tensor + int64_t total_elements = 1; + if (!test_case.empty() && test_case.num_inputs() > 0 && + test_case.inputs()[0].is_tensor()) { + const auto& sizes = test_case.inputs()[0].get_tensor_sizes(); + for (int64_t size : sizes) { + total_elements *= size; + } + } + + // Assume 1 FLOP per element for basic operations + return total_elements; +} + +BenchmarkGraph setup_compute_graph( + TestCase& test_case, + const std::string& op_name, + int op_invocations_per_execute) { + GraphConfig config; + config.enable_querypool = true; + // Pool sizing takes max(execute, prepack) * factor, so scaling the factor + // also over-reserves the prepack side (encoded once). Accepted: precise + // execute-only sizing would need runtime changes. + config.descriptor_pool_safety_factor *= + std::max(1, op_invocations_per_execute); + // Default-on (opt-out via TestCase::set_force_resize(false)): force every + // DynamicDispatchNode to run its resize function when execute_test_case + // runs propagate_resize() after prepack, exercising the op's resize formula + // even when input shapes are unchanged. + config.force_resize = test_case.get_force_resize(); + config.force_narrow_int4_tile = test_case.get_force_narrow_int4_tile(); + auto graph = std::make_unique(config); + + std::vector input_values; + + // Process input ValueSpecs + for (size_t i = 0; i < test_case.num_inputs(); ++i) { + const ValueSpec& input_spec = test_case.inputs()[i]; + + if (input_spec.is_none()) { + input_values.push_back(graph->add_none()); + } else if (input_spec.is_float()) { + ValueRef input_value = + graph->add_scalar(static_cast(input_spec.get_float_value())); + input_values.push_back(input_value); + } else if (input_spec.is_int()) { + ValueRef input_value = + graph->add_scalar(static_cast(input_spec.get_int_value())); + input_values.push_back(input_value); + } else if (input_spec.is_bool()) { + ValueRef input_value = graph->add_scalar(input_spec.get_bool_value()); + input_values.push_back(input_value); + } else if (input_spec.is_int_list()) { + // Convert int32_t list to int64_t list for ComputeGraph + const auto& int32_list = input_spec.get_int_list(); + std::vector int64_list; + int64_list.reserve(int32_list.size()); + for (int32_t val : int32_list) { + int64_list.push_back(static_cast(val)); + } + ValueRef input_value = graph->add_scalar_list(std::move(int64_list)); + input_values.push_back(input_value); + } else if (input_spec.is_string()) { + std::string str_copy = input_spec.get_string_value(); + ValueRef input_value = graph->add_string(std::move(str_copy)); + input_values.push_back(input_value); + } else if (input_spec.is_constant()) { + ValueRef input_value = graph->add_tensorref( + input_spec.get_tensor_sizes(), + input_spec.dtype, + input_spec.get_data_ptr()); + input_values.push_back(input_value); + } else { + IOValueRef input_io = graph->add_input_tensor( + input_spec.get_tensor_sizes(), + input_spec.dtype, + input_spec.storage_type, + input_spec.memory_layout); + input_values.push_back(input_io.value); + } + } + + std::vector output_values; + + // Process output ValueSpecs + for (size_t i = 0; i < test_case.num_outputs(); ++i) { + const ValueSpec& output_spec = test_case.outputs()[i]; + + if (!output_spec.is_tensor()) { + throw std::runtime_error("All output specifications must be tensors"); + } + + // Create output tensor + ValueRef output_value = graph->add_tensor( + output_spec.get_tensor_sizes(), + output_spec.dtype, + output_spec.storage_type, + output_spec.memory_layout); + + output_values.push_back(output_value); + } + + // Get the operator function and call it + auto opFn = VK_GET_OP_FN(op_name); + + // Create arguments vector for the operator function + std::vector op_args = input_values; + op_args.insert(op_args.end(), output_values.begin(), output_values.end()); + + // Nodes added before the op are staging uploads; nodes added after are + // staging downloads. Only the op's own nodes are repeated by benchmarks. + const size_t op_begin = graph->execute_nodes().size(); + opFn(*graph, op_args); + const size_t op_end = graph->execute_nodes().size(); + + for (size_t i = 0; i < output_values.size(); ++i) { + graph->set_output_value(output_values[i]); + } + return {std::move(graph), {op_begin, op_end}}; +} + +// Test execution utilities +BenchmarkResult execute_test_case( + TestCase& test_case, + int warmup_runs, + int benchmark_runs, + int chained_dispatches, + bool write_outputs) { + BenchmarkResult result( + test_case.name().empty() ? "unnamed_test_case" : test_case.name()); + + // Initialize querypool if using GPU timestamps + if (use_gpu_timestamps()) { + api::context()->initialize_querypool(); + } + + // Build the operator once. Benchmark repetition is encoded separately so + // persistent graph allocations are not duplicated. + BenchmarkGraph benchmark = setup_compute_graph( + test_case, test_case.operator_name(), chained_dispatches); + ComputeGraph& graph = *benchmark.graph; + + // Prepare the graph + graph.prepare(); + graph.prepack(); + + // Run resize functions once so force_resize exercises resize formulas even + // though the record/replay path below never calls propagate_resize(). + graph.propagate_resize(); + + // Copy input data into the graph's staging buffers + size_t graph_input_idx = 0; + for (size_t i = 0; i < test_case.num_inputs(); ++i) { + const ValueSpec& input_spec = test_case.inputs()[i]; + + // Only non-constant tensor inputs correspond to graph.inputs() entries + bool is_graph_input = input_spec.is_tensor() && !input_spec.is_constant() && + !input_spec.is_none(); + if (!is_graph_input) { + continue; + } + + if (graph_input_idx < graph.inputs().size()) { + const auto& input_ref = graph.inputs()[graph_input_idx]; + + // Get the appropriate data based on dtype + const void* data_ptr = nullptr; + size_t data_numel = input_spec.numel(); + + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (input_spec.dtype) { + case vkapi::kFloat: + data_ptr = input_spec.get_float_data().data(); + break; + case vkapi::kHalf: + data_ptr = input_spec.get_half_data().data(); + break; + case vkapi::kInt: + data_ptr = input_spec.get_int32_data().data(); + break; + case vkapi::kChar: + data_ptr = input_spec.get_int8_data().data(); + break; + case vkapi::kByte: + data_ptr = input_spec.get_uint8_data().data(); + break; + default: + throw std::runtime_error("Unsupported data type for input tensor"); + } + + // Copy data into staging buffer + graph.maybe_cast_and_copy_into_staging( + input_ref.staging, data_ptr, data_numel, input_spec.dtype); + } + ++graph_input_idx; + } + + RepeatedGraphExecutor graph_executor( + graph, chained_dispatches, benchmark.op_nodes); + + // Warmup runs + for (int run = 0; run < warmup_runs; ++run) { + graph_executor.execute(); + } + + // Benchmark runs - collect individual iteration timings + float total_cpu_time_us = 0.0f; + float total_gpu_time_us = 0.0f; + + const float chained_dispatches_f = static_cast(chained_dispatches); + + for (int run = 0; run < benchmark_runs; ++run) { + // Measure CPU time for each execute() call + auto cpu_start = std::chrono::high_resolution_clock::now(); + graph_executor.execute(); + auto cpu_end = std::chrono::high_resolution_clock::now(); + + auto cpu_duration = std::chrono::duration_cast( + cpu_end - cpu_start); + // Each execute() ran chained_dispatches op invocations; report + // per-invocation time. + float cpu_time_us = + static_cast(cpu_duration.count()) / chained_dispatches_f; + total_cpu_time_us += cpu_time_us; + + // Collect per-shader GPU timing - get raw shader results to preserve + // metadata + graph.context()->querypool().extract_results(); + const auto shader_results = + graph.context()->querypool().get_shader_timestamp_data(); + + // Calculate total GPU time from per-shader timings + float gpu_time_us = 0.0f; + for (const auto& shader_result : shader_results) { + // Check if this shader matches any filter pattern + bool filtered = false; + for (const auto& filter_pattern : test_case.get_shader_filter()) { + if (shader_result.kernel_name.find(filter_pattern) != + std::string::npos) { + filtered = true; + break; + } + } + + if (!filtered) { + uint64_t duration_ns = + shader_result.end_time_ns - shader_result.start_time_ns; + float duration_us = static_cast(duration_ns) / 1000.0f; + gpu_time_us += duration_us; + // Per-shader sample is already one querypool entry per dispatch (at + // chained_dispatches>1 we get that many entries per shader), so do NOT + // divide here — the aggregator (get_avg_time_us) averages across them + // naturally. + result.add_shader_timing( + shader_result.kernel_name, + duration_us, + shader_result.metadata.gwg, + shader_result.metadata.lwg); + } + } + // gpu_time_us aggregates chained_dispatches worth of shader runs; divide + // it down to per-invocation time. + gpu_time_us /= chained_dispatches_f; + total_gpu_time_us += gpu_time_us; + + // Add the appropriate timing based on the flag + float iter_time_us = use_gpu_timestamps() ? gpu_time_us : cpu_time_us; + result.add_iter_timing(iter_time_us); + } + + // Calculate averages for display + float avg_cpu_time_us = total_cpu_time_us / benchmark_runs; + float avg_gpu_time_us = total_gpu_time_us / benchmark_runs; + + // Print both timings if latency printing is enabled + if (print_latencies()) { + if (use_gpu_timestamps()) { + graph.context()->querypool().print_results(); + } + std::cout << " CPU timing: " << std::fixed << std::setprecision(3) + << avg_cpu_time_us << " μs" << std::endl; + std::cout << " GPU timing: " << std::fixed << std::setprecision(3) + << avg_gpu_time_us << " μs" << std::endl; + std::cout << " Using " << (use_gpu_timestamps() ? "GPU" : "CPU") + << " timing for result" << std::endl; + } + + // Copy output data from the graph's staging buffers. The benchmarking path + // skips this work since it doesn't need correctness data — the probe pass + // has already populated test_case.outputs() at chained_dispatches=1. + if (write_outputs) { + for (size_t i = 0; i < test_case.num_outputs(); ++i) { + ValueSpec& output_spec = test_case.outputs()[i]; + + if (output_spec.is_tensor() && i < graph.outputs().size()) { + const auto& output_ref = graph.outputs()[i]; + + // Ensure output data vector is properly sized + size_t data_numel = output_spec.numel(); + output_spec.resize_data(data_numel); + + // Get mutable data pointer for the output + void* data_ptr = output_spec.get_mutable_data_ptr(); + + if (data_ptr != nullptr) { + // Copy data from staging buffer to output spec + graph.maybe_cast_and_copy_from_staging( + output_ref.staging, data_ptr, data_numel, output_spec.dtype); + } + + // Print output tensor data if output printing is enabled + if (print_output()) { + std::string output_name = "Output[" + std::to_string(i) + "]"; + print_valuespec_data(output_spec, output_name); + } + } + } + } + + return result; +} + +TestResult execute_test_cases( + const std::function()>& test_case_generator, + const FlopCalculatorFunc& flop_calculator, + const std::string& operation_name, + int warmup_runs, + int benchmark_runs, + const ReferenceComputeFunc& reference_compute_func) { + TestResult results(operation_name); + + // Generate all test cases + std::vector test_cases = test_case_generator(); + + std::cout << "Executing " << test_cases.size() << " test cases for " + << operation_name << std::endl; + print_separator(); + + // Group test cases by ReferenceKey for caching reference computations + // Use a vector to preserve the order in which groups first appear + std::vector group_order; + std::unordered_map, ReferenceKeyHash> + groups; + for (size_t i = 0; i < test_cases.size(); ++i) { + ReferenceKey key = ReferenceKey::from_test_case(test_cases[i]); + if (groups.find(key) == groups.end()) { + group_order.push_back(key); + } + groups[key].push_back(i); + } + + bool any_correctness_failed = false; + float total_gflops = 0.0f; + size_t test_case_counter = 0; + + // Process each group: generate data, compute reference, execute, and print + // Iterate in the order groups first appeared in test_cases + for (const auto& key : group_order) { + const auto& indices = groups[key]; + if (indices.empty()) { + continue; + } + + // Get first test case as the "prototype" + size_t prototype_idx = indices[0]; + TestCase& prototype = test_cases[prototype_idx]; + + // Generate data for prototype with deterministic seed based on key + int group_seed = + static_cast(std::hash{}(key.key_string) % 10000); + for (auto& input : prototype.inputs()) { + input.ensure_data_generated(group_seed++); + } + + // Compute reference once for prototype + bool ref_computed = false; + if (reference_compute_func) { + try { + reference_compute_func(prototype); + ref_computed = true; + } catch (const std::invalid_argument&) { + // Reference computation skipped for this group + } + } + + // Copy data and reference to other test cases in group + for (size_t i = 1; i < indices.size(); ++i) { + size_t tc_idx = indices[i]; + TestCase& tc = test_cases[tc_idx]; + + // Copy input data from prototype + for (size_t j = 0; + j < tc.inputs().size() && j < prototype.inputs().size(); + ++j) { + auto& dest = tc.inputs()[j]; + const auto& src = prototype.inputs()[j]; + if (dest.is_tensor() && src.is_tensor() && dest.sizes == src.sizes && + dest.dtype == src.dtype) { + dest.share_data_from(src); + } + } + + // Copy reference output data if available + if (ref_computed) { + for (size_t j = 0; + j < tc.outputs().size() && j < prototype.outputs().size(); + ++j) { + const auto& src = prototype.outputs()[j]; + auto& dest = tc.outputs()[j]; + if (dest.is_tensor() && src.is_tensor() && dest.sizes == src.sizes && + dest.dtype == src.dtype) { + dest.share_reference_from(src); + } + } + } + } + + // Execute and print results for all test cases in this group + for (size_t tc_idx : indices) { + TestCase& test_case = test_cases[tc_idx]; + ++test_case_counter; + + // Execute single test case + BenchmarkResult result; + bool shader_not_supported = false; + try { + // Always run a probe pass at chained_dispatches=1 with + // write_outputs=true. This populates test_case.outputs() for the + // downstream correctness check with a clean single-dispatch result, + // and also gives us probe_us for adaptive sizing of the measurement + // run's chained_dispatches. + // + // probe_then_scale (Google Benchmark style): size chained_dispatches + // so each measurement execute() takes ~target_us. Tiny ops get a + // large factor (driving GPU governor escalation); heavy ops get a + // small factor (bounded wall time). Caveat: on Adreno 740 the probe + // runs at the DCVS-pinned clock (~220 MHz), so probe_us is inflated + // and the computed factor comes out under-sized for boost clock — but + // the default target is generous enough that an under-sized factor + // still drives sustained activity. + BenchmarkResult probe_result = execute_test_case( + test_case, + /*warmup_runs=*/1, + /*benchmark_runs=*/1, + /*chained_dispatches=*/1, + /*write_outputs=*/true); + float probe_us = probe_result.get_avg_time_us(); + if (probe_us < kMinProbeTimeUs) { + probe_us = kMinProbeTimeUs; + } + + int chained_dispatches; + const int manual_n = test_case.get_op_invocations_per_execute(); + if (manual_n > 0) { + chained_dispatches = manual_n; + } else { + const int target_us = test_case.get_target_execute_time_us(); + chained_dispatches = + std::max(1, static_cast(target_us / probe_us)); + chained_dispatches = + std::min(chained_dispatches, kMaxChainedDispatches); + } + if (debugging()) { + std::cout << "[probe] " << test_case.name() + << ": probe_us=" << probe_us + << ", chained_dispatches=" << chained_dispatches + << std::endl; + } + + // Measurement pass: chained_dispatches stacking, skip output copy + // since the probe already wrote the correctness data. + result = execute_test_case( + test_case, + warmup_runs, + benchmark_runs, + chained_dispatches, + /*write_outputs=*/false); + result.set_operator_name(test_case.operator_name()); + } catch (const vkcompute::vkapi::ShaderNotSupportedError&) { + result = BenchmarkResult( + test_case.name().empty() ? "unnamed_test_case" : test_case.name(), + test_case.operator_name()); + shader_not_supported = true; + } + + // Determine if this test case passed (has valid timing data) + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + + if (shader_not_supported) { + result.set_correctness_status(CorrectnessStatus::SKIPPED); + } else if (!vulkan_execute_succeeded) { + result.set_correctness_status(CorrectnessStatus::FAILED); + } else if (!ref_computed) { + result.set_correctness_status(CorrectnessStatus::SKIPPED); + } else { + // Reference function provided and succeeded - validate outputs + bool correctness_passed = true; + + for (size_t output_idx = 0; output_idx < test_case.num_outputs(); + ++output_idx) { + const ValueSpec& output_spec = test_case.outputs()[output_idx]; + + if (!output_spec.validate_against_reference( + test_case.get_abs_tolerance(), + test_case.get_rel_tolerance())) { + correctness_passed = false; + std::cout << " Correctness validation FAILED for test " + << result.get_kernel_name() << std::endl; + print_valuespec_data(output_spec, "vulkan output"); + print_valuespec_data(output_spec, "ref output", true); + } + } + + if (correctness_passed) { + result.set_correctness_status(CorrectnessStatus::PASSED); + } else { + any_correctness_failed = true; + result.set_correctness_status(CorrectnessStatus::FAILED); + } + } + + // Calculate GFLOPS for this test case using the provided FLOP calculator + float case_gflops = 0.0f; + if (vulkan_execute_succeeded) { + // Use the provided FLOP calculator to get total FLOPs for this test + // case + int64_t total_flops = flop_calculator(test_case); + float flops = static_cast(total_flops); + float avg_time_us = result.get_avg_time_us(); + if (avg_time_us > 0.0f && total_flops > 0) { + case_gflops = (flops / 1e9f) / (avg_time_us / 1e6f); + } + + total_gflops += case_gflops; + } else { + case_gflops = -1.0f; // Indicate failure + } + + // Calculate tensor info for display + std::string size_info = "["; + if (!test_case.empty() && test_case.num_inputs() > 0 && + test_case.inputs()[0].is_tensor()) { + const auto& sizes = test_case.inputs()[0].get_tensor_sizes(); + for (size_t j = 0; j < sizes.size(); ++j) { + size_info += std::to_string(sizes[j]); + if (j < sizes.size() - 1) { + size_info += "x"; + } + } + } + size_info += "]"; + + // Print progress using the BenchmarkResult member function + result.print_summary( + utils::safe_downcast(test_case_counter), size_info, case_gflops); + + // Add result to collection + results.add_result(std::move(result)); + + test_case.clear(); + } + } + + // Set the overall results on the TestResult + results.set_correctness_passed(!any_correctness_failed); + results.set_gflops(total_gflops); + + print_separator(); + std::cout << "Completed " << results.size() << " test cases" << std::endl; + + // Hard-fail if any correctness check failed. The per-case loop above keeps + // running after a mismatch (so the full pass/fail matrix and per-16x16-tile + // mismatch maps are printed for debugging) rather than aborting on the first + // failure; throwing here preserves the original abort-on-mismatch behavior. + if (any_correctness_failed) { + throw std::runtime_error("Correctness validation failed"); + } + + return results; +} + +// Convenience overload that uses the default FLOP calculator +TestResult execute_test_cases( + const std::function()>& test_case_generator, + const std::string& operation_name, + int warmup_runs, + int benchmark_runs, + const ReferenceComputeFunc& reference_compute_func) { + return execute_test_cases( + test_case_generator, + default_flop_calculator, + operation_name, + warmup_runs, + benchmark_runs, + reference_compute_func); +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/runner.h b/backends/vulkan/test/ops/benchmark/framework/runner.h new file mode 100644 index 00000000000..b3493aed9cb --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/runner.h @@ -0,0 +1,123 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include "results.h" +#include "test_case.h" + +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// +// Test case execution +// + +using FlopCalculatorFunc = std::function; + +// Default FLOP calculation function (assumes 1 FLOP per element) +int64_t default_flop_calculator(const TestCase& test_case); + +using ReferenceComputeFunc = std::function; + +// Half-open index range of the operator's own dispatch nodes within a +// benchmark graph's execute_nodes(). Staging upload nodes precede it, staging +// download nodes follow it. +struct OpNodeRange { + size_t begin = 0; + size_t end = 0; +}; + +// A benchmark graph plus the location of its repeatable operator nodes. The +// graph is heap-held: ComputeGraph owns its Context and must never be moved +// (a moved-from graph's destructor dereferences a null context). +struct BenchmarkGraph { + std::unique_ptr graph; + OpNodeRange op_nodes; +}; + +// Benchmark-only executor that records a graph's execute nodes into a single +// reusable command buffer, then replays it on every execute(). Staging uploads +// are encoded once, operator nodes N times, staging downloads once, so each +// iteration performs the same work as the old stacked-nodes layout (1 upload + +// N ops + 1 download) and the per-invocation divisor is unchanged. Production +// ComputeGraph::execute() behavior is unchanged. +// +// Notes for interpreting benchmark numbers: +// - Submit granularity differs from stacking N distinct nodes: all encodings +// live in one command buffer with one submit per iteration (the old path +// could split across command buffers at the node-count threshold), so +// per-dispatch times may shift systematically against older data. +// - Resize functions run once via propagate_resize() in execute_test_case +// before recording; replay itself never re-triggers resize. +// - Repeated encodings share one node/dispatch id, so per-repetition +// querypool attribution is unavailable (aggregation keys on kernel name). +class RepeatedGraphExecutor final { + public: + RepeatedGraphExecutor( + ComputeGraph& graph, + int repetitions, + OpNodeRange op_nodes); + void execute(); + + private: + ComputeGraph& graph_; + std::unique_ptr command_; +}; + +// Runs a measurement at the given chained_dispatches factor. The operator is +// built once, then its execute nodes are encoded that many times into a +// benchmark-only reusable command buffer. The probe-then-scale orchestration +// lives in execute_test_cases(). +// +// write_outputs controls whether the graph's staging output buffers are copied +// back into test_case.outputs() at the end of the run. The probe path needs +// write_outputs=true so the correctness check has a clean single-dispatch +// reference. The benchmarking path passes write_outputs=false to avoid the +// per-iter GPU->CPU copy cost. +BenchmarkResult execute_test_case( + TestCase& test_case, + int warmup_runs = 1, + int benchmark_runs = 1, + int chained_dispatches = 1, + bool write_outputs = true); + +TestResult execute_test_cases( + const std::function()>& test_case_generator, + const FlopCalculatorFunc& flop_calculator, + const std::string& operation_name = "Operation", + int warmup_runs = 1, + int benchmark_runs = 1, + const ReferenceComputeFunc& reference_compute_func = nullptr); + +TestResult execute_test_cases( + const std::function()>& test_case_generator, + const std::string& operation_name = "Operation", + int warmup_runs = 1, + int benchmark_runs = 1, + const ReferenceComputeFunc& reference_compute_func = nullptr); + +// Setup compute graph based on TestCase and operation name. The op function is +// invoked once. op_invocations_per_execute is used only to reserve enough +// descriptor capacity for benchmark-only repeated command encoding. Returns +// the graph plus the range of the operator's own nodes for repeated encoding. +BenchmarkGraph setup_compute_graph( + TestCase& test_case, + const std::string& op_name, + int op_invocations_per_execute = 1); + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/test_case.cpp b/backends/vulkan/test/ops/benchmark/framework/test_case.cpp new file mode 100644 index 00000000000..550bd10c136 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/test_case.cpp @@ -0,0 +1,79 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "test_case.h" + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// ReferenceKey implementation +ReferenceKey ReferenceKey::from_test_case(const TestCase& tc) { + std::ostringstream oss; + + // Serialize inputs that affect reference computation + // Skip: storage_type, memory_layout, string values (like impl_selector) + for (size_t i = 0; i < tc.inputs().size(); ++i) { + const ValueSpec& input = tc.inputs()[i]; + oss << "i" << i << ":"; + + if (input.is_tensor()) { + // For tensors: sizes, dtype, data_gen_type, is_constant + oss << "T["; + for (size_t j = 0; j < input.sizes.size(); ++j) { + if (j > 0) { + oss << ","; + } + oss << input.sizes[j]; + } + oss << "]d" << static_cast(input.dtype); + oss << "g" << static_cast(input.data_gen_type); + oss << "c" << (input.is_constant() ? 1 : 0); + oss << "n" << (input.is_none() ? 1 : 0); + } else if (input.is_int()) { + oss << "I" << input.get_int_value(); + } else if (input.is_float()) { + oss << "F" << input.get_float_value(); + } else if (input.is_bool()) { + oss << "B" << (input.get_bool_value() ? 1 : 0); + } else if (input.is_int_list()) { + oss << "L["; + const auto& list = input.get_int_list(); + for (size_t j = 0; j < list.size(); ++j) { + if (j > 0) { + oss << ","; + } + oss << list[j]; + } + oss << "]"; + } + // Skip string inputs (like impl_selector) as they don't affect reference + oss << ";"; + } + + // Also include output shapes for completeness + for (size_t i = 0; i < tc.outputs().size(); ++i) { + const ValueSpec& output = tc.outputs()[i]; + oss << "o" << i << ":["; + for (size_t j = 0; j < output.sizes.size(); ++j) { + if (j > 0) { + oss << ","; + } + oss << output.sizes[j]; + } + oss << "]d" << static_cast(output.dtype) << ";"; + } + + ReferenceKey key; + key.key_string = oss.str(); + return key; +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/test_case.h b/backends/vulkan/test/ops/benchmark/framework/test_case.h new file mode 100644 index 00000000000..5edd5838ffc --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/test_case.h @@ -0,0 +1,236 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include "value_spec.h" + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// +// ReferenceKey for caching reference computations +// + +// Captures the identity of input conditions for test case grouping. +// Test cases with the same ReferenceKey should produce identical reference +// outputs, so reference computation can be cached and reused. +struct ReferenceKey { + std::string key_string; + + static ReferenceKey from_test_case(const class TestCase& tc); + + bool operator==(const ReferenceKey& other) const { + return key_string == other.key_string; + } +}; + +struct ReferenceKeyHash { + size_t operator()(const ReferenceKey& k) const { + return std::hash{}(k.key_string); + } +}; + +// +// Shader filter defaults +// + +// Default shader filter: excludes layout conversions and quantization overhead +// This is used for tests where quantize/dequantize are overhead operations +inline const std::vector kDefaultShaderFilter = { + "nchw_to", + "to_nchw", + "quantize_and_pack_4w4c", + "unpack_4w4c_and_dequantize"}; + +// Layout-only filter: only excludes layout conversions +// Use this for tests where quantize/dequantize ARE the operations being tested +inline const std::vector kLayoutOnlyShaderFilter = { + "nchw_to", + "to_nchw"}; + +// +// TestCase +// + +// Default per-execute() wall-clock target used by the probe-then-scale logic +// in execute_test_cases(). Picked generously enough that even an under-sized +// chained_dispatches factor (the probe runs at governor-pinned clock and +// underestimates the boost-clock latency) still drives sustained GPU +// activity during measurement. +constexpr int kDefaultTargetExecuteTimeUs = 100000; + +class TestCase { + public: + TestCase() + : abs_tolerance_(2e-3f), + rel_tolerance_(1e-3f), + shader_filter_(kDefaultShaderFilter) {} + explicit TestCase(const std::string& name) + : name_(name), + abs_tolerance_(2e-3f), + rel_tolerance_(1e-3f), + shader_filter_(kDefaultShaderFilter) {} + + void set_name(const std::string& name) { + name_ = name; + } + const std::string& name() const { + return name_; + } + + void set_operator_name(const std::string& op_name) { + operator_name_ = op_name; + } + const std::string& operator_name() const { + return operator_name_; + } + + // Tolerance settings + void set_abs_tolerance(float abs_tolerance) { + abs_tolerance_ = abs_tolerance; + } + float get_abs_tolerance() const { + return abs_tolerance_; + } + + void set_rel_tolerance(float rel_tolerance) { + rel_tolerance_ = rel_tolerance; + } + float get_rel_tolerance() const { + return rel_tolerance_; + } + + // Shader filter settings - list of shader name patterns to exclude from + // timing + void set_shader_filter(const std::vector& filter) { + shader_filter_ = filter; + } + const std::vector& get_shader_filter() const { + return shader_filter_; + } + + // Manual override for the number of chained dispatches per measurement + // iteration (a.k.a. chained_dispatches). If > 0, the framework uses this + // directly and skips the probe phase. 0 (the default) means adaptive + // (probe-then-scale). + void set_op_invocations_per_execute(int n) { + op_invocations_per_execute_ = n; + } + int get_op_invocations_per_execute() const { + return op_invocations_per_execute_; + } + + // Target single-execute duration in microseconds. Used only when the + // manual chained_dispatches override is not set. Default + // kDefaultTargetExecuteTimeUs, picked generously enough to mitigate Adreno + // DCVS governor pinning during the probe. + void set_target_execute_time_us(int us) { + target_execute_time_us_ = us; + } + int get_target_execute_time_us() const { + return target_execute_time_us_; + } + + // When true, the ComputeGraph built for this test case sets + // GraphConfig::force_resize, so every DynamicDispatchNode runs its resize + // function once during measurement setup (execute_test_case runs + // propagate_resize() after prepack) even when no input shape changed. + // Because the output is already allocated at the swept shape, the resize + // must recompute the same shape from the current input — a wrong resize + // formula resizes the output to a mismatched shape and surfaces as a test + // failure. Default true (opt-out): every custom_ops test exercises its + // resize formulas across the swept shapes. Call set_force_resize(false) for + // the rare op whose resize fn is intentionally not shape-preserving under a + // fixed output allocation. + void set_force_resize(bool force_resize) { + force_resize_ = force_resize; + } + bool get_force_resize() const { + return force_resize_; + } + + // When true, the ComputeGraph sets GraphConfig::force_narrow_int4_tile, so + // the int4 linear picks its narrow (2-row) output tile on any GPU. That + // variant is otherwise reachable only on Mali, which no CI runner has. + void set_force_narrow_int4_tile(bool force) { + force_narrow_int4_tile_ = force; + } + bool get_force_narrow_int4_tile() const { + return force_narrow_int4_tile_; + } + + void add_input_spec(const ValueSpec& spec) { + inputs_.push_back(spec); + } + + const std::vector& inputs() const { + return inputs_; + } + + std::vector& inputs() { + return inputs_; + } + + size_t num_inputs() const { + return inputs_.size(); + } + + void add_output_spec(const ValueSpec& spec) { + outputs_.push_back(spec); + } + + const std::vector& outputs() const { + return outputs_; + } + + std::vector& outputs() { + return outputs_; + } + + size_t num_outputs() const { + return outputs_.size(); + } + + bool empty() const { + return inputs_.empty() && outputs_.empty(); + } + void clear() { + inputs_.clear(); + outputs_.clear(); + name_.clear(); + operator_name_.clear(); + abs_tolerance_ = 2e-3f; + rel_tolerance_ = 1e-3f; + shader_filter_ = kDefaultShaderFilter; + op_invocations_per_execute_ = 0; + target_execute_time_us_ = kDefaultTargetExecuteTimeUs; + force_resize_ = true; + force_narrow_int4_tile_ = false; + } + + private: + std::string name_; + std::string operator_name_; + std::vector inputs_; + std::vector outputs_; + float abs_tolerance_; + float rel_tolerance_; + std::vector shader_filter_; + int op_invocations_per_execute_ = 0; // 0 = adaptive + int target_execute_time_us_ = kDefaultTargetExecuteTimeUs; + bool force_resize_ = true; + bool force_narrow_int4_tile_ = false; +}; + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_coopmat_probe.cpp b/backends/vulkan/test/ops/benchmark/framework/test_coopmat_probe.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/test_coopmat_probe.cpp rename to backends/vulkan/test/ops/benchmark/framework/test_coopmat_probe.cpp diff --git a/backends/vulkan/test/ops/benchmark/framework/utils.h b/backends/vulkan/test/ops/benchmark/framework/utils.h new file mode 100644 index 00000000000..51d5fd1bd8f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/utils.h @@ -0,0 +1,27 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "labels.h" +#include "results.h" +#include "runner.h" +#include "test_case.h" +#include "value_spec.h" +#include "weight_utils.h" diff --git a/backends/vulkan/test/ops/benchmark/framework/value_spec.cpp b/backends/vulkan/test/ops/benchmark/framework/value_spec.cpp new file mode 100644 index 00000000000..b9752596f89 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/value_spec.cpp @@ -0,0 +1,1024 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "value_spec.h" +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +int get_seed() { + static int seed = 42; + return seed++; +} + +int get_seed_or_explicit(int explicit_seed) { + if (explicit_seed >= 0) { + return explicit_seed; + } + return get_seed(); +} + +// Forward declarations for data generation utilities +void generate_random_float_data( + std::vector& data, + float min_val = -1.0f, + float max_val = 1.0f, + int explicit_seed = -1); +void generate_random_int_data( + std::vector& data, + int min_val = -10, + int max_val = 10, + int explicit_seed = -1); +void generate_randint_float_data( + std::vector& data, + int min_val = -10, + int max_val = 10, + int explicit_seed = -1); +void generate_randint_half_data( + std::vector& data, + int min_val = -10, + int max_val = 10, + int explicit_seed = -1); +void generate_random_int8_data( + std::vector& data, + int8_t min_val = -10, + int8_t max_val = 10, + int explicit_seed = -1); +void generate_random_uint8_data( + std::vector& data, + uint8_t min_val = 0, + uint8_t max_val = 255, + int explicit_seed = -1); +void generate_random_2xint4_data( + std::vector& data, + int explicit_seed = -1); +void generate_random_2xint4_data( + std::vector& data, + int explicit_seed = -1); +void generate_random_int4_data( + std::vector& data, + int8_t min_val = -8, + int8_t max_val = 7, + int explicit_seed = -1); +void generate_ones_data(std::vector& data); +void generate_zeros_data(std::vector& data); + +// Convert a float32 value to IEEE 754 half-precision (uint16_t) +uint16_t float_to_half(float value) { + uint32_t float_bits; + std::memcpy(&float_bits, &value, sizeof(float)); + uint32_t sign = (float_bits >> 31) & 0x1; + int32_t exponent = static_cast((float_bits >> 23) & 0xFF) - 127; + uint32_t mantissa = float_bits & 0x7FFFFF; + + uint16_t half_val; + if (exponent > 15) { + half_val = static_cast((sign << 15) | 0x7C00); // Inf + } else if (exponent < -14) { + half_val = static_cast(sign << 15); // Zero / subnormal + } else { + half_val = static_cast( + (sign << 15) | (static_cast(exponent + 15) << 10) | + (mantissa >> 13)); + } + return half_val; +} + +// Convert a IEEE 754 half-precision (uint16_t) value to float32 +float half_to_float(uint16_t half_val) { + uint32_t sign = (half_val >> 15) & 0x1; + uint32_t exponent = (half_val >> 10) & 0x1F; + uint32_t mantissa = half_val & 0x3FF; + + float result; + if (exponent == 0) { + result = std::ldexp(static_cast(mantissa), -24); + } else if (exponent == 31) { + result = mantissa ? std::numeric_limits::quiet_NaN() + : std::numeric_limits::infinity(); + } else { + result = std::ldexp( + 1.0f + static_cast(mantissa) / 1024.0f, exponent - 15); + } + if (sign) { + result = -result; + } + return result; +} + +// ValueSpec implementation +void ValueSpec::ensure_unique_data() const { + if (data_.use_count() != 1) { + data_ = std::make_shared(*data_); + } +} + +void ValueSpec::ensure_unique_reference_data() const { + if (reference_data_.use_count() != 1) { + reference_data_ = std::make_shared(*reference_data_); + } +} + +void ValueSpec::generate_tensor_data(int seed) const { + if (spec_type != SpecType::Tensor) { + return; + } + + ensure_unique_data(); + auto& float_data = data_->float_data; + auto& int32_data = data_->int32_data; + auto& half_data = data_->half_data; + auto& int8_data = data_->int8_data; + auto& uint8_data = data_->uint8_data; + + int64_t num_elements = numel(); + + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: { + float_data.resize(num_elements); + if (data_gen_type == DataGenType::RANDOM) { + generate_random_float_data(float_data, -1.0f, 1.0f, seed); + } else if (data_gen_type == DataGenType::RANDOM_SCALES) { + generate_random_float_data(float_data, 0.005, 0.015, seed); + } else if (data_gen_type == DataGenType::RANDINT) { + generate_randint_float_data(float_data, -10, 10, seed); + } else if (data_gen_type == DataGenType::RANDINT8) { + generate_randint_float_data(float_data, -128, 127, seed); + } else if (data_gen_type == DataGenType::RANDINT4) { + generate_randint_float_data(float_data, -8, 7, seed); + } else if (data_gen_type == DataGenType::ONES) { + generate_ones_data(float_data); + } else if (data_gen_type == DataGenType::ZEROS) { + generate_zeros_data(float_data); + } else { + generate_zeros_data(float_data); + } + break; + } + case vkapi::kHalf: { + half_data.resize(num_elements); + if (data_gen_type == DataGenType::RANDOM) { + // Generate random float data first, then convert to IEEE 754 half. + std::vector temp_data(num_elements); + generate_random_float_data(temp_data, -1.0f, 1.0f, seed); + for (size_t i = 0; i < temp_data.size(); ++i) { + half_data[i] = float_to_half(temp_data[i]); + } + } else if (data_gen_type == DataGenType::RANDOM_SCALES) { + // Generate random scales in float, then convert to proper fp16 + std::vector temp_data(num_elements); + generate_random_float_data(temp_data, 0.005f, 0.015f, seed); + for (size_t i = 0; i < temp_data.size(); ++i) { + half_data[i] = float_to_half(temp_data[i]); + } + } else if (data_gen_type == DataGenType::RANDINT) { + generate_randint_half_data(half_data, -10, 10, seed); + } else if (data_gen_type == DataGenType::RANDINT8) { + generate_randint_half_data(half_data, -128, 127, seed); + } else if (data_gen_type == DataGenType::RANDINT4) { + generate_randint_half_data(half_data, -8, 7, seed); + } else if (data_gen_type == DataGenType::ONES) { + std::fill(half_data.begin(), half_data.end(), float_to_half(1.0f)); + } else if (data_gen_type == DataGenType::ZEROS) { + std::fill( + half_data.begin(), + half_data.end(), + static_cast(0)); // 0.0 in half + } else { + std::fill( + half_data.begin(), + half_data.end(), + static_cast(0)); // 0.0 in half + } + break; + } + case vkapi::kInt: { + int32_data.resize(num_elements); + if (data_gen_type == DataGenType::RANDOM) { + generate_random_int_data(int32_data, -10, 10, seed); + } else if (data_gen_type == DataGenType::RANDINT) { + generate_random_int_data( + int32_data, + -10, + 10, + seed); // For int type, RANDINT is same as RANDOM + } else if (data_gen_type == DataGenType::RANDINT8) { + generate_random_int_data(int32_data, -128, 127, seed); + } else if (data_gen_type == DataGenType::RANDINT4) { + generate_random_int_data(int32_data, -8, 7, seed); + } else if (data_gen_type == DataGenType::ONES) { + std::fill(int32_data.begin(), int32_data.end(), 1); + } else if (data_gen_type == DataGenType::ZEROS) { + std::fill(int32_data.begin(), int32_data.end(), 0); + } else { + std::fill(int32_data.begin(), int32_data.end(), 0); + } + break; + } + case vkapi::kChar: { + int8_data.resize(num_elements); + if (data_gen_type == DataGenType::RANDOM) { + generate_random_int8_data(int8_data, -10, 10, seed); + } else if (data_gen_type == DataGenType::RANDINT) { + generate_random_int8_data(int8_data, -10, 10, seed); + } else if (data_gen_type == DataGenType::RANDINT8) { + generate_random_int8_data(int8_data, -128, 127, seed); + } else if (data_gen_type == DataGenType::RANDINT4) { + generate_random_2xint4_data(int8_data, seed); + } else if (data_gen_type == DataGenType::ONES) { + std::fill(int8_data.begin(), int8_data.end(), 1); + } else if (data_gen_type == DataGenType::ONES_INT4) { + int8_t packed_data = (1 << 4) | 1; + std::fill(int8_data.begin(), int8_data.end(), packed_data); + } else if (data_gen_type == DataGenType::ZEROS) { + std::fill(int8_data.begin(), int8_data.end(), 0); + } else { + std::fill(int8_data.begin(), int8_data.end(), 0); + } + break; + } + case vkapi::kByte: { + uint8_data.resize(num_elements); + if (data_gen_type == DataGenType::RANDOM) { + generate_random_uint8_data(uint8_data, 0, 255, seed); + } else if (data_gen_type == DataGenType::RANDINT) { + generate_random_uint8_data(uint8_data, 0, 255, seed); + } else if (data_gen_type == DataGenType::RANDINT8) { + generate_random_uint8_data(uint8_data, 0, 255, seed); + } else if (data_gen_type == DataGenType::RANDINT4) { + generate_random_2xint4_data(uint8_data, seed); + } else if (data_gen_type == DataGenType::ONES) { + std::fill(uint8_data.begin(), uint8_data.end(), 1); + } else if (data_gen_type == DataGenType::ONES_INT4) { + uint8_t packed_data = (9 << 4) | 9; + std::fill(uint8_data.begin(), uint8_data.end(), packed_data); + } else if (data_gen_type == DataGenType::ZEROS) { + std::fill(uint8_data.begin(), uint8_data.end(), 0); + } else { + std::fill(uint8_data.begin(), uint8_data.end(), 0); + } + break; + } + default: + // Default to float + float_data.resize(num_elements); + if (data_gen_type == DataGenType::RANDOM) { + generate_random_float_data(float_data, -1.0f, 1.0f, seed); + } else if (data_gen_type == DataGenType::RANDINT) { + generate_randint_float_data(float_data, -10, 10, seed); + } else if (data_gen_type == DataGenType::ONES) { + generate_ones_data(float_data); + } else if (data_gen_type == DataGenType::ZEROS) { + generate_zeros_data(float_data); + } else { + generate_zeros_data(float_data); + } + break; + } +} + +int64_t ValueSpec::numel() const { + if (spec_type == SpecType::Int || spec_type == SpecType::Float || + spec_type == SpecType::Bool) { + return 1; + } else if (spec_type == SpecType::IntList) { + return sizes.empty() ? 0 : sizes[0]; + } else { // Tensor + int64_t total = 1; + for (int64_t size : sizes) { + total *= size; + } + return total; + } +} + +size_t ValueSpec::nbytes() const { + size_t element_size = 0; + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + element_size = sizeof(float); + break; + case vkapi::kHalf: + element_size = sizeof(uint16_t); + break; + case vkapi::kInt: + element_size = sizeof(int32_t); + break; + case vkapi::kChar: + element_size = sizeof(int8_t); + break; + case vkapi::kByte: + element_size = sizeof(uint8_t); + break; + default: + element_size = sizeof(float); // Default fallback + break; + } + return numel() * element_size; +} + +std::string ValueSpec::to_string() const { + std::string result = "ValueSpec("; + + switch (spec_type) { + case SpecType::Tensor: + result += "type=Tensor, sizes=["; + break; + case SpecType::IntList: + result += "type=IntList, count="; + result += std::to_string(sizes.empty() ? 0 : sizes[0]); + result += ", data_gen="; + result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; + result += ")"; + return result; + case SpecType::Int: + result += "type=Int, value="; + result += std::to_string(get_int_value()); + result += ", data_gen="; + result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; + result += ")"; + return result; + case SpecType::Float: + result += "type=Float, value="; + result += std::to_string(get_float_value()); + result += ", data_gen="; + result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; + result += ")"; + return result; + case SpecType::Bool: + result += "type=Bool, value="; + result += get_bool_value() ? "true" : "false"; + result += ", data_gen="; + result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; + result += ")"; + return result; + case SpecType::String: + result += "type=String, value=\""; + result += get_string_value(); + result += "\")"; + return result; + } + + for (size_t i = 0; i < sizes.size(); ++i) { + result += std::to_string(sizes[i]); + if (i < sizes.size() - 1) { + result += ", "; + } + } + result += "]"; + + if (spec_type == SpecType::Tensor) { + result += ", dtype="; + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + result += "float"; + break; + case vkapi::kHalf: + result += "half"; + break; + case vkapi::kInt: + result += "int32"; + break; + case vkapi::kChar: + result += "int8"; + break; + case vkapi::kByte: + result += "uint8"; + break; + default: + result += "unknown"; + break; + } + + result += ", memory_layout="; + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (memory_layout) { + case utils::kWidthPacked: + result += "WidthPacked"; + break; + case utils::kHeightPacked: + result += "HeightPacked"; + break; + case utils::kChannelsPacked: + result += "ChannelsPacked"; + break; + default: + result += "unknown"; + break; + } + + result += ", storage_type="; + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (storage_type) { + case utils::kTexture3D: + result += "Texture3D"; + break; + case utils::kBuffer: + result += "Buffer"; + break; + default: + result += "unknown"; + break; + } + } + + result += ", data_gen="; + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (data_gen_type) { + case DataGenType::FIXED: + result += "FIXED"; + break; + case DataGenType::RANDOM: + result += "RANDOM"; + break; + case DataGenType::RANDINT: + result += "RANDINT"; + break; + case DataGenType::RANDINT8: + result += "RANDINT8"; + break; + case DataGenType::RANDINT4: + result += "RANDINT4"; + break; + case DataGenType::ONES: + result += "ONES"; + break; + case DataGenType::ZEROS: + result += "ZEROS"; + break; + default: + result += "unknown"; + break; + } + result += ")"; + return result; +} + +// Additional ValueSpec methods +void ValueSpec::resize_data(size_t new_size) { + // Generate first so a deferred tensor keeps its data-gen pattern (resized, + // not pinned to zeros by the data_generated_ flag set below). + ensure_data_generated(); + ensure_unique_data(); + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + data_->float_data.resize(new_size); + break; + case vkapi::kHalf: + data_->half_data.resize(new_size); + break; + case vkapi::kInt: + data_->int32_data.resize(new_size); + break; + case vkapi::kChar: + data_->int8_data.resize(new_size); + break; + case vkapi::kByte: + data_->uint8_data.resize(new_size); + break; + default: + data_->float_data.resize(new_size); + break; + } + data_generated_ = true; +} + +void* ValueSpec::get_mutable_data_ptr() { + ensure_data_generated(); + ensure_unique_data(); + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + return data_->float_data.data(); + case vkapi::kHalf: + return data_->half_data.data(); + case vkapi::kInt: + return data_->int32_data.data(); + case vkapi::kChar: + return data_->int8_data.data(); + case vkapi::kByte: + return data_->uint8_data.data(); + default: + return data_->float_data.data(); + } +} + +float ValueSpec::get_element(size_t index) const { + ensure_data_generated(); + if (index >= static_cast(numel())) { + return 0.0f; + } + + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + return index < data_->float_data.size() ? data_->float_data[index] : 0.0f; + case vkapi::kHalf: + return index < data_->half_data.size() + ? half_to_float(data_->half_data[index]) + : 0.0f; + case vkapi::kInt: + return index < data_->int32_data.size() + ? static_cast(data_->int32_data[index]) + : 0.0f; + case vkapi::kChar: + return index < data_->int8_data.size() + ? static_cast(data_->int8_data[index]) + : 0.0f; + case vkapi::kByte: + return index < data_->uint8_data.size() + ? static_cast(data_->uint8_data[index]) + : 0.0f; + default: + return 0.0f; + } +} + +const void* ValueSpec::get_data_ptr() const { + ensure_data_generated(); + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + return data_->float_data.data(); + case vkapi::kHalf: + return data_->half_data.data(); + case vkapi::kInt: + return data_->int32_data.data(); + case vkapi::kChar: + return data_->int8_data.data(); + case vkapi::kByte: + return data_->uint8_data.data(); + default: + throw std::runtime_error("Unsupported data type for get_data_ptr"); + } +} + +void generate_random_float_data( + std::vector& data, + float min_val, + float max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_real_distribution dis(min_val, max_val); + for (auto& val : data) { + val = dis(gen); + } +} + +void generate_random_int_data( + std::vector& data, + int min_val, + int max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(min_val, max_val); + for (auto& val : data) { + val = dis(gen); + } +} + +void generate_randint_float_data( + std::vector& data, + int min_val, + int max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(min_val, max_val); + for (auto& val : data) { + val = static_cast(dis(gen)); + } +} + +void generate_randint_half_data( + std::vector& data, + int min_val, + int max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(min_val, max_val); + for (auto& val : data) { + val = float_to_half(static_cast(dis(gen))); + } +} + +void generate_ones_data(std::vector& data) { + std::fill(data.begin(), data.end(), 1.0f); +} + +void generate_random_int8_data( + std::vector& data, + int8_t min_val, + int8_t max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(min_val, max_val); + for (auto& val : data) { + val = static_cast(dis(gen)); + } +} + +void generate_random_uint8_data( + std::vector& data, + uint8_t min_val, + uint8_t max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(min_val, max_val); + for (auto& val : data) { + val = static_cast(dis(gen)); + } +} + +void generate_random_int4_data( + std::vector& data, + int8_t min_val, + int8_t max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(min_val, max_val); + for (auto& val : data) { + val = static_cast(dis(gen)); + } +} + +void generate_random_2xint4_data(std::vector& data, int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(-8, 7); // Signed 4-bit range + for (auto& val : data) { + // Generate two separate 4-bit values + int8_t lower_4bits = static_cast(dis(gen)) & 0x0F; + int8_t upper_4bits = static_cast(dis(gen)) & 0x0F; + // Pack them into a single 8-bit value + val = (upper_4bits << 4) | lower_4bits; + } +} + +void generate_random_2xint4_data( + std::vector& data, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(0, 15); // Unsigned 4-bit range + for (auto& val : data) { + // Generate two separate 4-bit values + uint8_t lower_4bits = static_cast(dis(gen)) & 0x0F; + uint8_t upper_4bits = static_cast(dis(gen)) & 0x0F; + // Pack them into a single 8-bit value + val = (upper_4bits << 4) | lower_4bits; + } +} + +void generate_zeros_data(std::vector& data) { + std::fill(data.begin(), data.end(), 0.0f); +} + +// Correctness checking against reference data +bool ValueSpec::validate_against_reference( + float abs_tolerance, + float rel_tolerance) const { + // Only validate float and half tensors. For half tensors, convert the + // computed half data to float for comparison against the fp32 reference. + if (!is_tensor() || (dtype != vkapi::kFloat && dtype != vkapi::kHalf)) { + return true; // Skip validation for non-float/half or non-tensor types + } + + // For kHalf, materialize the GPU output as float so the same tolerance + // machinery can compare against the (always-float) reference data. + std::vector half_as_float; + if (dtype == vkapi::kHalf) { + const auto& half_bits = get_half_data(); + half_as_float.resize(half_bits.size()); + for (size_t i = 0; i < half_bits.size(); ++i) { + half_as_float[i] = half_to_float(half_bits[i]); + } + } + // Materialize computed data as float32 for comparison. The dtype is + // guaranteed to be float or half by the early-out above. + const std::vector& computed_data = + (dtype == vkapi::kHalf) ? half_as_float : get_float_data(); + const auto& reference_data = get_ref_float_data(); + + // Skip validation if no reference data is available + if (reference_data.empty()) { + return true; + } + + // Check if sizes match + if (computed_data.size() != reference_data.size()) { + if (debugging()) { + std::cout << "Size mismatch: computed=" << computed_data.size() + << ", reference=" << reference_data.size() << std::endl; + } + return false; + } + + // Element-wise comparison with both absolute and relative tolerance + size_t num_mismatched = 0; + size_t first_mismatch = 0; + for (size_t i = 0; i < computed_data.size(); ++i) { + float diff = std::abs(computed_data[i] - reference_data[i]); + float abs_ref = std::abs(reference_data[i]); + + // Check if either absolute or relative tolerance condition is satisfied + bool abs_tolerance_ok = diff <= abs_tolerance; + bool rel_tolerance_ok = diff <= rel_tolerance * abs_ref; + + if (!abs_tolerance_ok && !rel_tolerance_ok) { + if (num_mismatched == 0) { + first_mismatch = i; + std::cout << "Mismatch at element " << i + << ": computed=" << computed_data[i] + << ", reference=" << reference_data[i] << ", diff=" << diff + << ", abs_tolerance=" << abs_tolerance + << ", rel_tolerance=" << rel_tolerance + << ", rel_threshold=" << (rel_tolerance * abs_ref) + << std::endl; + } + num_mismatched++; + } + } + if (num_mismatched > 0) { + std::cout << " total mismatched: " << num_mismatched << " / " + << computed_data.size() << " (first at " << first_mismatch << ")" + << std::endl; + // For 2D outputs, print a per-16x16-tile mismatch-count map to expose + // the spatial structure of the failure (e.g. zeroed MMA subtiles). + if (sizes.size() == 2) { + const int64_t Mr = sizes[0]; + const int64_t Nc = sizes[1]; + std::cout << " 16x16-tile mismatch counts (rows=M/16, cols=N/16):" + << std::endl; + for (int64_t ti = 0; ti < (Mr + 15) / 16; ++ti) { + std::cout << " "; + for (int64_t tj = 0; tj < (Nc + 15) / 16; ++tj) { + int count = 0; + for (int64_t r = ti * 16; r < std::min(Mr, (ti + 1) * 16); ++r) { + for (int64_t c = tj * 16; c < std::min(Nc, (tj + 1) * 16); ++c) { + float diff = std::abs( + computed_data[r * Nc + c] - reference_data[r * Nc + c]); + float abs_ref = std::abs(reference_data[r * Nc + c]); + if (diff > abs_tolerance && diff > rel_tolerance * abs_ref) { + count++; + } + } + } + std::cout << std::setw(4) << count; + } + std::cout << std::endl; + } + } + return false; + } + + if (debugging()) { + std::cout << "Correctness validation PASSED" << std::endl; + } + return true; +} + +// Ensure data is generated for this ValueSpec +void ValueSpec::ensure_data_generated(int seed) const { + if (data_generated_) { + return; + } + generate_tensor_data(seed); + data_generated_ = true; +} + +void ValueSpec::share_data_from(const ValueSpec& other) { + if (!is_tensor() || !other.is_tensor()) { + return; + } + // Materialize the source first: sharing an ungenerated payload would let a + // later access materialize each spec independently under different seeds. + other.ensure_data_generated(); + data_ = other.data_; + data_generated_ = true; +} + +void ValueSpec::share_reference_from(const ValueSpec& other) { + if (!is_tensor() || !other.is_tensor()) { + return; + } + reference_data_ = other.reference_data_; +} + +// ValueSpec data printing utilities +void print_valuespec_data( + const ValueSpec& spec, + const std::string& name, + const bool print_ref_data, + size_t max_elements, + int precision) { + std::cout << "\n" << name << " Data:" << std::endl; + std::cout << " Type: " << spec.to_string() << std::endl; + + if (!spec.is_tensor()) { + if (spec.is_int()) { + std::cout << " Value: " << spec.get_int_value() << std::endl; + } else if (spec.is_int_list()) { + const auto& int_list = spec.get_int_list(); + std::cout << " Values: ["; + size_t print_count = std::min(max_elements, int_list.size()); + for (size_t i = 0; i < print_count; ++i) { + std::cout << int_list[i]; + if (i < print_count - 1) { + std::cout << ", "; + } + } + if (int_list.size() > max_elements) { + std::cout << ", ... (" << (int_list.size() - max_elements) << " more)"; + } + std::cout << "]" << std::endl; + } + return; + } + + // Print tensor data + size_t total_elements = spec.numel(); + size_t print_count = std::min(max_elements, total_elements); + + std::cout << " Total elements: " << total_elements << std::endl; + std::cout << " Data (first " << print_count << " elements): ["; + + std::cout << std::fixed << std::setprecision(precision); + + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (spec.dtype) { + case vkapi::kFloat: { + auto data = spec.get_float_data().data(); + if (print_ref_data) { + data = spec.get_ref_float_data().data(); + } + for (size_t i = 0; i < print_count; ++i) { + std::cout << data[i]; + if (i < print_count - 1) { + std::cout << ", "; + } + } + break; + } + case vkapi::kHalf: { + if (print_ref_data) { + const auto& ref = spec.get_ref_float_data(); + for (size_t i = 0; i < print_count; ++i) { + std::cout << ref[i]; + if (i < print_count - 1) { + std::cout << ", "; + } + } + break; + } + const auto& data = spec.get_half_data(); + for (size_t i = 0; i < print_count; ++i) { + // Convert IEEE 754 half-precision bit pattern back to float. + float value = half_to_float(data[i]); + std::cout << value; + if (i < print_count - 1) { + std::cout << ", "; + } + } + break; + } + case vkapi::kInt: { + const auto& data = spec.get_int32_data(); + for (size_t i = 0; i < print_count; ++i) { + std::cout << data[i]; + if (i < print_count - 1) { + std::cout << ", "; + } + } + break; + } + case vkapi::kChar: { + const auto& data = spec.get_int8_data(); + if (spec.is_int4()) { + // Print each 4-bit value individually + size_t element_count = 0; + for (size_t i = 0; i < data.size() && element_count < print_count; + ++i) { + // Extract lower 4 bits (signed) + int8_t lower_4bits = data[i] & 0x0F; + if (lower_4bits > 7) { + lower_4bits -= 16; // Convert to signed + } + std::cout << static_cast(lower_4bits); + element_count++; + + if (element_count < print_count) { + std::cout << ", "; + // Extract upper 4 bits (signed) + int8_t upper_4bits = (data[i] >> 4) & 0x0F; + if (upper_4bits > 7) { + upper_4bits -= 16; // Convert to signed + } + std::cout << static_cast(upper_4bits); + element_count++; + + if (element_count < print_count) { + std::cout << ", "; + } + } + } + } else { + for (size_t i = 0; i < print_count; ++i) { + std::cout << static_cast(data[i]); + if (i < print_count - 1) { + std::cout << ", "; + } + } + } + break; + } + case vkapi::kByte: { + const auto& data = spec.get_uint8_data(); + if (spec.is_int4()) { + // Print each 4-bit value individually + size_t element_count = 0; + for (size_t i = 0; i < data.size() && element_count < print_count; + ++i) { + // Extract lower 4 bits + uint8_t lower_4bits = data[i] & 0x0F; + std::cout << static_cast(lower_4bits); + element_count++; + + if (element_count < print_count) { + std::cout << ", "; + // Extract upper 4 bits + uint8_t upper_4bits = (data[i] >> 4) & 0x0F; + std::cout << static_cast(upper_4bits); + element_count++; + + if (element_count < print_count) { + std::cout << ", "; + } + } + } + } else { + for (size_t i = 0; i < print_count; ++i) { + std::cout << static_cast(data[i]); + if (i < print_count - 1) { + std::cout << ", "; + } + } + } + break; + } + default: + std::cout << "unsupported data type"; + break; + } + + if (total_elements > max_elements) { + std::cout << ", ... (" << (total_elements - max_elements) << " more)"; + } + std::cout << "]" << std::endl; + + // Print some statistics for tensor data + if (total_elements > 0) { + float min_val = 0.0f, max_val = 0.0f, sum = 0.0f; + bool first = true; + + for (size_t i = 0; i < total_elements; ++i) { + float val = spec.get_element(i); + if (first) { + min_val = max_val = val; + first = false; + } else { + min_val = std::min(min_val, val); + max_val = std::max(max_val, val); + } + sum += val; + } + + float mean = sum / total_elements; + std::cout << " Statistics: min=" << std::setprecision(precision) << min_val + << ", max=" << max_val << ", mean=" << mean << ", sum=" << sum + << std::endl; + } +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/value_spec.h b/backends/vulkan/test/ops/benchmark/framework/value_spec.h new file mode 100644 index 00000000000..ebb87e2c323 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/value_spec.h @@ -0,0 +1,404 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +using namespace vkcompute; + +// +// ValueSpec class +// + +enum class SpecType { Tensor, IntList, Int, Float, Bool, String }; + +// Data generation types +enum class DataGenType { + FIXED, + RANDOM, + RANDOM_SCALES, + RANDINT, + RANDINT8, + RANDINT4, + ONES, + ONES_INT4, + ZEROS +}; + +// Value specification struct +struct ValueSpec { + std::vector sizes; + vkapi::ScalarType dtype; + utils::GPUMemoryLayout memory_layout; + utils::StorageType storage_type; + SpecType spec_type; + DataGenType data_gen_type; + bool is_constant_tensor; + bool is_none_flag; + bool is_int4_tensor; + std::string string_data; + + ValueSpec( + const std::vector& sizes, + vkapi::ScalarType dtype, + utils::StorageType storage_type = utils::kTexture3D, + utils::GPUMemoryLayout memory_layout = utils::kWidthPacked) + : sizes(sizes), + dtype(dtype), + memory_layout(memory_layout), + storage_type(storage_type), + spec_type(SpecType::Tensor), + data_gen_type(DataGenType::ZEROS), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(false) { + // Data generation is deferred until first access (any data getter or + // ensure_data_generated() triggers it). + } + + // Constructor for tensor with custom data generation type + ValueSpec( + const std::vector& sizes, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout, + DataGenType data_gen_type) + : sizes(sizes), + dtype(dtype), + memory_layout(memory_layout), + storage_type(storage_type), + spec_type(SpecType::Tensor), + data_gen_type(data_gen_type), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(false) { + // Data generation is deferred until first access (any data getter or + // ensure_data_generated() triggers it). + } + + // Constructor for single int + ValueSpec(int32_t value) + : sizes({1}), + dtype(vkapi::kInt), + memory_layout(utils::kWidthPacked), + storage_type(utils::kTexture3D), + spec_type(SpecType::Int), + data_gen_type(DataGenType::FIXED), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(true) { + data_->int32_data.push_back(value); + } + + // Constructor for single float + ValueSpec(float value) + : sizes({1}), + dtype(vkapi::kFloat), + memory_layout(utils::kWidthPacked), + storage_type(utils::kTexture3D), + spec_type(SpecType::Float), + data_gen_type(DataGenType::FIXED), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(true) { + data_->float_data.push_back(value); + } + + // Constructor for single bool + ValueSpec(bool value) + : sizes({1}), + dtype(vkapi::kInt), + memory_layout(utils::kWidthPacked), + storage_type(utils::kTexture3D), + spec_type(SpecType::Bool), + data_gen_type(DataGenType::FIXED), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(true) { + data_->int32_data.push_back(value ? 1 : 0); + } + + // Constructor for int list + ValueSpec(const std::vector& values) + : sizes({static_cast(values.size())}), + dtype(vkapi::kInt), + memory_layout(utils::kWidthPacked), + storage_type(utils::kTexture3D), + spec_type(SpecType::IntList), + data_gen_type(DataGenType::FIXED), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(true) { + data_->int32_data = values; + } + + // Factory method for string (avoids ambiguity with vector constructor) + static ValueSpec make_string(const std::string& value) { + ValueSpec spec; + spec.sizes = {1}; + spec.dtype = vkapi::kInt; + spec.memory_layout = utils::kWidthPacked; + spec.storage_type = utils::kTexture3D; + spec.spec_type = SpecType::String; + spec.data_gen_type = DataGenType::FIXED; + spec.is_constant_tensor = false; + spec.is_none_flag = false; + spec.is_int4_tensor = false; + spec.data_generated_ = true; + spec.string_data = value; + return spec; + } + + // Default constructor + ValueSpec() + : dtype(vkapi::kFloat), + memory_layout(utils::kWidthPacked), + storage_type(utils::kTexture3D), + spec_type(SpecType::Tensor), + data_gen_type(DataGenType::ZEROS), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(false) {} + + int64_t numel() const; + size_t nbytes() const; + std::string to_string() const; + + bool is_tensor() const { + return spec_type == SpecType::Tensor; + } + bool is_int_list() const { + return spec_type == SpecType::IntList; + } + bool is_int() const { + return spec_type == SpecType::Int; + } + bool is_float() const { + return spec_type == SpecType::Float; + } + bool is_bool() const { + return spec_type == SpecType::Bool; + } + bool is_string() const { + return spec_type == SpecType::String; + } + + int32_t get_int_value() const { + ensure_data_generated(); + return data_->int32_data.empty() ? 0 : data_->int32_data[0]; + } + float get_float_value() const { + ensure_data_generated(); + return data_->float_data.empty() ? 0.0f : data_->float_data[0]; + } + bool get_bool_value() const { + ensure_data_generated(); + return data_->int32_data.empty() ? false : (data_->int32_data[0] != 0); + } + const std::string& get_string_value() const { + return string_data; + } + const std::vector& get_int_list() const { + ensure_data_generated(); + return data_->int32_data; + } + const std::vector& get_tensor_sizes() const { + return sizes; + } + + // References and pointers into tensor data must not be held across any other + // access to the same spec: a mutable access may detach the shared payload, + // leaving a previously returned reference bound to the old payload. Consume + // immediately. + const std::vector& get_float_data() const { + ensure_data_generated(); + return data_->float_data; + } + const std::vector& get_int32_data() const { + ensure_data_generated(); + return data_->int32_data; + } + const std::vector& get_half_data() const { + ensure_data_generated(); + return data_->half_data; + } + const std::vector& get_int8_data() const { + ensure_data_generated(); + return data_->int8_data; + } + const std::vector& get_uint8_data() const { + ensure_data_generated(); + return data_->uint8_data; + } + + std::vector& get_float_data() { + ensure_data_generated(); + ensure_unique_data(); + return data_->float_data; + } + std::vector& get_int32_data() { + ensure_data_generated(); + ensure_unique_data(); + return data_->int32_data; + } + std::vector& get_half_data() { + ensure_data_generated(); + ensure_unique_data(); + return data_->half_data; + } + std::vector& get_int8_data() { + ensure_data_generated(); + ensure_unique_data(); + return data_->int8_data; + } + std::vector& get_uint8_data() { + ensure_data_generated(); + ensure_unique_data(); + return data_->uint8_data; + } + + const std::vector& get_ref_float_data() const { + return reference_data_->float_data; + } + const std::vector& get_ref_int32_data() const { + return reference_data_->int32_data; + } + const std::vector& get_ref_half_data() const { + return reference_data_->half_data; + } + const std::vector& get_ref_int8_data() const { + return reference_data_->int8_data; + } + const std::vector& get_ref_uint8_data() const { + return reference_data_->uint8_data; + } + + std::vector& get_ref_float_data() { + ensure_unique_reference_data(); + return reference_data_->float_data; + } + std::vector& get_ref_int32_data() { + ensure_unique_reference_data(); + return reference_data_->int32_data; + } + std::vector& get_ref_half_data() { + ensure_unique_reference_data(); + return reference_data_->half_data; + } + std::vector& get_ref_int8_data() { + ensure_unique_reference_data(); + return reference_data_->int8_data; + } + std::vector& get_ref_uint8_data() { + ensure_unique_reference_data(); + return reference_data_->uint8_data; + } + + void resize_data(size_t new_size); + void* get_mutable_data_ptr(); + float get_element(size_t index) const; + + // Data generation methods for deferred generation and caching. + // + // ValueSpec is not thread-safe: lazy materialization and copy-on-write + // detach mutate shared state from const methods. Test cases are built and + // executed on a single thread. + // + // Implicit materialization (any data getter, resize_data) consumes the + // global seed counter. Callers needing deterministic data must call + // ensure_data_generated(explicit_seed) before any other access; a later + // seeded call is a no-op once data is generated. + bool is_data_generated() const { + return data_generated_; + } + void ensure_data_generated(int seed = -1) const; + void share_data_from(const ValueSpec& other); + void share_reference_from(const ValueSpec& other); + + // Set/get constant flag + bool is_constant() const { + return is_constant_tensor; + } + void set_constant(bool is_constant) { + is_constant_tensor = is_constant; + } + + // Set/get none flag + bool is_none() const { + return is_none_flag; + } + + void set_none(bool is_none) { + is_none_flag = is_none; + } + + // Set/get int4 flag + bool is_int4() const { + return is_int4_tensor; + } + void set_int4(bool is_int4) { + is_int4_tensor = is_int4; + } + + const void* get_data_ptr() const; + + // Correctness checking against reference data + // Returns true if computed data matches reference data within tolerance + // Only validates float tensors as specified in requirements + bool validate_against_reference( + float abs_tolerance = 2e-3f, + float rel_tolerance = 1e-3f) const; + + private: + struct TensorData { + std::vector float_data; + std::vector int32_data; + std::vector half_data; + std::vector int8_data; + std::vector uint8_data; + }; + + void ensure_unique_data() const; + void ensure_unique_reference_data() const; + void generate_tensor_data(int seed = -1) const; + + mutable bool data_generated_ = false; + mutable std::shared_ptr data_ = std::make_shared(); + mutable std::shared_ptr reference_data_ = + std::make_shared(); +}; + +void print_valuespec_data( + const ValueSpec& spec, + const std::string& name = "ValueSpec", + const bool print_ref_data = false, + size_t max_elements = 20, + int precision = 6); + +// Half-precision conversion utilities +uint16_t float_to_half(float value); +float half_to_float(uint16_t half_val); + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/weight_utils.cpp b/backends/vulkan/test/ops/benchmark/framework/weight_utils.cpp new file mode 100644 index 00000000000..5161178d978 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/weight_utils.cpp @@ -0,0 +1,247 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "weight_utils.h" +#include "config.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +ValueRef quantized_weights_canvas( + ComputeGraph& graph, + const ValueRef weight_ref) { + const auto original_sizes = graph.sizes_of(weight_ref); + + std::vector sorted_sizes = original_sizes; + std::sort(sorted_sizes.begin(), sorted_sizes.end(), std::greater()); + int64_t largest1 = sorted_sizes.size() > 0 ? sorted_sizes[0] : 0; + + std::vector final_sizes = {1, largest1, largest1}; + + // Debug logging if debugging flag is set + if (debugging()) { + std::cout << "Debug: Creating quantized weights canvas tensor" << std::endl; + std::cout << "Debug: Original sizes: ["; + for (size_t i = 0; i < original_sizes.size(); ++i) { + std::cout << original_sizes[i]; + if (i < original_sizes.size() - 1) + std::cout << ", "; + } + std::cout << "]" << std::endl; + std::cout << "Debug: Canvas sizes: ["; + for (size_t i = 0; i < final_sizes.size(); ++i) { + std::cout << final_sizes[i]; + if (i < final_sizes.size() - 1) + std::cout << ", "; + } + std::cout << "]" << std::endl; + } + + ValueRef packed_weight = graph.add_tensor( + final_sizes, vkapi::kInt, utils::kTexture3D, utils::kWidthPacked); + + std::string kernel_name = "packed_int32_canvas"; + add_storage_type_suffix(kernel_name, graph.storage_type_of(packed_weight)); + + graph.prepack_nodes().emplace_back(new PrepackNode( + graph, + VK_KERNEL_FROM_STR(kernel_name), + graph.create_gwg(packed_weight), + graph.create_lwg(packed_weight), + weight_ref, + packed_weight, + // UBOs + {graph.logical_limits_ubo(packed_weight)}, + // Specialization constants + {}, + // Push Constants + {})); + + return packed_weight; +} + +ValueRef float_tensor_canvas(ComputeGraph& graph, const ValueRef weight_ref) { + const auto original_sizes = graph.sizes_of(weight_ref); + + std::vector sorted_sizes = original_sizes; + std::sort(sorted_sizes.begin(), sorted_sizes.end(), std::greater()); + int64_t largest1 = sorted_sizes.size() > 0 ? sorted_sizes[0] : 0; + + std::vector final_sizes = {1, largest1, largest1}; + + // Debug logging if debugging flag is set + if (debugging()) { + std::cout << "Debug: Creating float tensor canvas" << std::endl; + std::cout << "Debug: Original sizes: ["; + for (size_t i = 0; i < original_sizes.size(); ++i) { + std::cout << original_sizes[i]; + if (i < original_sizes.size() - 1) + std::cout << ", "; + } + std::cout << "]" << std::endl; + std::cout << "Debug: Canvas sizes: ["; + for (size_t i = 0; i < final_sizes.size(); ++i) { + std::cout << final_sizes[i]; + if (i < final_sizes.size() - 1) + std::cout << ", "; + } + std::cout << "]" << std::endl; + } + + ValueRef packed_weight = graph.add_tensor( + final_sizes, vkapi::kFloat, utils::kTexture3D, utils::kWidthPacked); + + graph.prepack_nodes().emplace_back(new PrepackNode( + graph, + VK_KERNEL_FROM_STR("float_canvas"), + graph.create_gwg(packed_weight), + graph.create_lwg(packed_weight), + weight_ref, + packed_weight, + // UBOs + {graph.logical_limits_ubo(packed_weight)}, + // Specialization constants + {}, + // Push Constants + {})); + + return packed_weight; +} + +// Compute weight sums for quantized operations (linear and convolution) +void compute_weight_sums( + ValueSpec& weight_sums, + const ValueSpec& quantized_weight, + int64_t out_features, + int64_t elements_per_output_feature) { + auto& weight_sums_data = weight_sums.get_int32_data(); + auto& quantized_weight_data = quantized_weight.get_int8_data(); + + // Don't resize down - the buffer may be pre-allocated with aligned size. + // Only resize up if needed. + if (weight_sums_data.size() < static_cast(out_features)) { + weight_sums_data.resize(out_features); + } + + // For each output feature, compute the sum of quantized weights + for (int64_t out_f = 0; out_f < out_features; ++out_f) { + int32_t sum = 0; + for (int64_t elem = 0; elem < elements_per_output_feature; ++elem) { + // Weight indexing depends on the layout: + // For linear: [out_features, in_features] -> out_f * + // elements_per_output_feature + elem For conv2d: [C_out, C_in * K_h * + // K_w] -> out_f * elements_per_output_feature + elem + int64_t weight_idx = out_f * elements_per_output_feature + elem; + sum += static_cast(quantized_weight_data[weight_idx]); + } + weight_sums_data[out_f] = sum; + } +} + +// Compute weight sums for 4D quantized conv2d operations +// Weight layout: [C_out, K_h, K_w, align_up_4(C_in_per_group)] +void compute_weight_sums_4d( + ValueSpec& weight_sums, + const ValueSpec& quantized_weight, + int64_t out_channels, + int64_t kernel_h, + int64_t kernel_w, + int64_t aligned_in_channels) { + auto& weight_sums_data = weight_sums.get_int32_data(); + auto& quantized_weight_data = quantized_weight.get_int8_data(); + + weight_sums_data.resize(out_channels); + + // For each output channel, compute the sum of quantized weights + for (int64_t out_c = 0; out_c < out_channels; ++out_c) { + int32_t sum = 0; + + for (int64_t kh = 0; kh < kernel_h; ++kh) { + for (int64_t kw = 0; kw < kernel_w; ++kw) { + for (int64_t in_c = 0; in_c < aligned_in_channels; ++in_c) { + // Weight indexing: [out_c, kh, kw, in_c] + int64_t weight_idx = + out_c * (kernel_h * kernel_w * aligned_in_channels) + + kh * (kernel_w * aligned_in_channels) + kw * aligned_in_channels + + in_c; + sum += static_cast(quantized_weight_data[weight_idx]); + } + } + } + + weight_sums_data[out_c] = sum; + } +} + +// Helper function to unpack 4-bit values from uint8 (same as in +// q4gsw_linear.cpp) +std::pair unpack_4bit_utils(uint8_t packed) { + // Extract lower 4 bits and upper 4 bits + int8_t lower = packed & 0x0F; + int8_t upper = (packed >> 4) & 0x0F; + + // Subtract 8 from unpacked 4-bit values + lower -= 8; + upper -= 8; + + return std::make_pair(lower, upper); +} + +// Compute weight sums for 4-bit group symmetric quantized weights +void compute_weight_sums_4bit_grouped( + ValueSpec& weight_sums, + const ValueSpec& quantized_weight, + int64_t num_groups, + int64_t out_features, + int64_t group_size) { + auto& weight_sums_data = weight_sums.get_int32_data(); + auto& quantized_weight_data = quantized_weight.get_uint8_data(); + + // Resize to [num_groups, out_features] + weight_sums_data.resize(num_groups * out_features); + + // For each group and each output feature, compute the sum of quantized + // weights in that group + for (int64_t group_idx = 0; group_idx < num_groups; ++group_idx) { + for (int64_t out_f = 0; out_f < out_features; ++out_f) { + int32_t sum = 0; + + // Sum weights for this group and output feature + for (int64_t in_group = 0; in_group < group_size; ++in_group) { + int64_t in_f = group_idx * group_size + in_group; + + // Get packed weight value - weight matrix is [N, K/2] + int64_t weight_idx = + out_f * ((num_groups * group_size) / 2) + (in_f / 2); + uint8_t packed_weight = quantized_weight_data[weight_idx]; + + // Unpack 4-bit weight + auto unpacked = unpack_4bit_utils(packed_weight); + int8_t weight_4bit = (in_f % 2 == 0) ? unpacked.first : unpacked.second; + + sum += static_cast(weight_4bit); + } + + // Store sum for this group and output feature + int64_t sums_idx = group_idx * out_features + out_f; + weight_sums_data[sums_idx] = sum; + } + } +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/weight_utils.h b/backends/vulkan/test/ops/benchmark/framework/weight_utils.h new file mode 100644 index 00000000000..7c9319703b9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/weight_utils.h @@ -0,0 +1,52 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include "value_spec.h" + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +ValueRef quantized_weights_canvas( + ComputeGraph& graph, + const ValueRef weight_ref); + +ValueRef float_tensor_canvas(ComputeGraph& graph, const ValueRef weight_ref); + +// Compute weight sums for quantized operations (linear and convolution) +void compute_weight_sums( + ValueSpec& weight_sums, + const ValueSpec& quantized_weight, + int64_t out_features, + int64_t elements_per_output_feature); + +// Compute weight sums for 4D quantized conv2d operations +// Weight layout: [C_out, K_h, K_w, align_up_4(C_in_per_group)] +void compute_weight_sums_4d( + ValueSpec& weight_sums, + const ValueSpec& quantized_weight, + int64_t out_channels, + int64_t kernel_h, + int64_t kernel_w, + int64_t aligned_in_channels); + +// Compute weight sums for 4-bit group symmetric quantized weights +void compute_weight_sums_4bit_grouped( + ValueSpec& weight_sums, + const ValueSpec& quantized_weight, + int64_t num_groups, + int64_t out_features, + int64_t group_size); + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/targets.bzl b/backends/vulkan/test/ops/benchmark/targets.bzl new file mode 100644 index 00000000000..ec70bb549f1 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/targets.bzl @@ -0,0 +1,129 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") +load( + "@fbsource//xplat/executorch/backends/vulkan:targets.bzl", + "get_platforms", + "vulkan_spv_shader_lib", +) + +def define_custom_op_test_binary(custom_op_name, subdir, extra_deps = [], include_torch = False): + deps_list = [ + ":prototyping_utils", + ":operator_implementations", + ":custom_ops_shaderlib", + "//executorch/backends/vulkan:vulkan_graph_runtime", + ] + ([runtime.external_dep_location("libtorch")] if include_torch else []) + extra_deps + + runtime.cxx_binary( + name = custom_op_name, + srcs = [ + "{}/{}.cpp".format(subdir, custom_op_name), + ], + platforms = get_platforms(), + define_static_target = False, + deps = deps_list, + ) + +def define_common_targets(is_fbcode = False): + if is_fbcode: + return + + # Shader library from GLSL files + runtime.filegroup( + name = "custom_ops_shaders", + srcs = native.glob([ + "test_ops/glsl/**/*", + ]), + visibility = ["PUBLIC"], + ) + + vulkan_spv_shader_lib( + name = "custom_ops_shaderlib", + spv_filegroups = { + ":custom_ops_shaders": "test_ops/glsl", + }, + is_fbcode = is_fbcode, + ) + + # Prototyping utilities library + runtime.cxx_library( + name = "prototyping_utils", + srcs = [ + "framework/config.cpp", + "framework/conv2d_utils.cpp", + "framework/results.cpp", + "framework/runner.cpp", + "framework/test_case.cpp", + "framework/value_spec.cpp", + "framework/weight_utils.cpp", + ], + headers = [ + "framework/config.h", + "framework/conv2d_utils.h", + "framework/labels.h", + "framework/results.h", + "framework/runner.h", + "framework/test_case.h", + "framework/utils.h", + "framework/value_spec.h", + "framework/weight_utils.h", + ], + exported_headers = [ + "framework/config.h", + "framework/conv2d_utils.h", + "framework/labels.h", + "framework/results.h", + "framework/runner.h", + "framework/test_case.h", + "framework/utils.h", + "framework/value_spec.h", + "framework/weight_utils.h", + ], + platforms = get_platforms(), + deps = [ + "//executorch/backends/vulkan:vulkan_graph_runtime", + ], + visibility = ["PUBLIC"], + ) + + # Operator implementations library + runtime.cxx_library( + name = "operator_implementations", + srcs = native.glob([ + "test_ops/impl/*.cpp", + ]), + platforms = get_platforms(), + deps = [ + "//executorch/backends/vulkan:vulkan_graph_runtime", + ":custom_ops_shaderlib", + ], + visibility = ["PUBLIC"], + link_whole = True, + ) + + + + + define_custom_op_test_binary("test_add", "cases/add") + define_custom_op_test_binary("test_q8csw_linear", "cases/q8csw_linear") + define_custom_op_test_binary("test_q8csw_conv2d", "cases/q8csw_conv2d") + define_custom_op_test_binary("test_choose_qparams_per_row", "cases/choose_qparams_per_row") + define_custom_op_test_binary("test_q4gsw_linear", "cases/q4gsw_linear") + define_custom_op_test_binary("test_q8ta_qdq", "cases/q8ta/qdq") + define_custom_op_test_binary("test_q8ta_clone", "cases/q8ta/clone") + define_custom_op_test_binary("test_q8ta_binary", "cases/q8ta/binary") + define_custom_op_test_binary("test_q8ta_conv2d", "cases/q8ta/conv2d") + define_custom_op_test_binary("test_q8ta_conv2d_pw", "cases/q8ta/conv2d") + define_custom_op_test_binary("test_q8ta_conv2d_dw", "cases/q8ta/conv2d") + define_custom_op_test_binary("test_q8ta_linear", "cases/q8ta/linear") + define_custom_op_test_binary("test_q8ta_conv2d_transposed", "cases/q8ta/conv2d_transposed") + define_custom_op_test_binary("test_q8ta_pixel_shuffle", "cases/q8ta/pixel_shuffle") + define_custom_op_test_binary("test_q8ta_unary", "cases/q8ta/unary") + define_custom_op_test_binary("test_mm", "cases/mm") + define_custom_op_test_binary("test_conv2d", "cases/conv2d") + define_custom_op_test_binary("test_conv2d_pw", "cases/conv2d") + define_custom_op_test_binary("test_conv2d_dw", "cases/conv2d") + define_custom_op_test_binary("test_embedding_q4gsw", "cases/embedding_q4gsw") + define_custom_op_test_binary("test_conv1d_pw", "cases/conv1d") + define_custom_op_test_binary("test_conv1d_dw", "cases/conv1d") + define_custom_op_test_binary("test_fpa_q4gsw_linear", "cases/q4gsw_linear") + define_custom_op_test_binary("test_sdpa", "cases/sdpa") diff --git a/backends/vulkan/test/custom_ops/glsl/add.yaml b/backends/vulkan/test/ops/benchmark/test_ops/glsl/add.yaml similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/add.yaml rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/add.yaml diff --git a/backends/vulkan/test/custom_ops/glsl/add_buffer.glsl b/backends/vulkan/test/ops/benchmark/test_ops/glsl/add_buffer.glsl similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/add_buffer.glsl rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/add_buffer.glsl diff --git a/backends/vulkan/test/custom_ops/glsl/add_texture.glsl b/backends/vulkan/test/ops/benchmark/test_ops/glsl/add_texture.glsl similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/add_texture.glsl rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/add_texture.glsl diff --git a/backends/vulkan/test/custom_ops/glsl/float_canvas.glsl b/backends/vulkan/test/ops/benchmark/test_ops/glsl/float_canvas.glsl similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/float_canvas.glsl rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/float_canvas.glsl diff --git a/backends/vulkan/test/custom_ops/glsl/packed_int32_canvas_buffer.glsl b/backends/vulkan/test/ops/benchmark/test_ops/glsl/packed_int32_canvas_buffer.glsl similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/packed_int32_canvas_buffer.glsl rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/packed_int32_canvas_buffer.glsl diff --git a/backends/vulkan/test/custom_ops/glsl/packed_int32_canvas_texture3d.glsl b/backends/vulkan/test/ops/benchmark/test_ops/glsl/packed_int32_canvas_texture3d.glsl similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/packed_int32_canvas_texture3d.glsl rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/packed_int32_canvas_texture3d.glsl diff --git a/backends/vulkan/test/custom_ops/glsl/q4gsw_linear_gemv__w_4x8.glsl b/backends/vulkan/test/ops/benchmark/test_ops/glsl/q4gsw_linear_gemv__w_4x8.glsl similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/q4gsw_linear_gemv__w_4x8.glsl rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/q4gsw_linear_gemv__w_4x8.glsl diff --git a/backends/vulkan/test/custom_ops/glsl/q4gsw_linear_gemv__w_4x8.yaml b/backends/vulkan/test/ops/benchmark/test_ops/glsl/q4gsw_linear_gemv__w_4x8.yaml similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/q4gsw_linear_gemv__w_4x8.yaml rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/q4gsw_linear_gemv__w_4x8.yaml diff --git a/backends/vulkan/test/custom_ops/impl/AddPrototype.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/AddPrototype.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/AddPrototype.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/AddPrototype.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestConv1dDW.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv1dDW.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestConv1dDW.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv1dDW.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestConv1dPW.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv1dPW.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestConv1dPW.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv1dPW.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestConv2d.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2d.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestConv2d.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2d.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestConv2dDw.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2dDw.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestConv2dDw.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2dDw.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestConv2dPw.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2dPw.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestConv2dPw.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2dPw.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestFpaQ4gswLinear.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestFpaQ4gswLinear.cpp similarity index 99% rename from backends/vulkan/test/custom_ops/impl/TestFpaQ4gswLinear.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestFpaQ4gswLinear.cpp index 2da2cd6ceb7..4b41391b2cd 100644 --- a/backends/vulkan/test/custom_ops/impl/TestFpaQ4gswLinear.cpp +++ b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestFpaQ4gswLinear.cpp @@ -212,10 +212,10 @@ LocalWorkGroup pick_q4gsw_coop_lwg( } // Spec-constant LWG values for the q4gsw_linear_gemv__w_4x8_nc[_nosg] shaders -// (now shipped from test/custom_ops/glsl/). The sg variant pins subgroupSize -// to 64 via VK_EXT_subgroup_size_control; the nosg variant uses shared-mem -// reduction so the lane count is purely an LWG choice. Both use 4 subgroups -// per workgroup. +// (now shipped from test/ops/benchmark/test_ops/glsl/). The sg variant pins +// subgroupSize to 64 via VK_EXT_subgroup_size_control; the nosg variant uses +// shared-mem reduction so the lane count is purely an LWG choice. Both use 4 +// subgroups per workgroup. constexpr uint32_t kGemvSubgroupSize = 64u; constexpr uint32_t kGemvNumSubgroups = 4u; diff --git a/backends/vulkan/test/custom_ops/impl/TestMatmulLinear.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestMatmulLinear.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestMatmulLinear.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestMatmulLinear.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taBinary.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taBinary.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taBinary.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taBinary.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taClone.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taClone.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taClone.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taClone.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taConv2d.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taConv2d.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taConv2d.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taConv2d.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taConv2dTransposed.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taConv2dTransposed.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taConv2dTransposed.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taConv2dTransposed.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taLinear.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taLinear.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taLinear.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taLinear.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taPixelShuffle.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taPixelShuffle.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taPixelShuffle.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taPixelShuffle.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taQuantizeDequantize.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taQuantizeDequantize.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taQuantizeDequantize.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taQuantizeDequantize.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taUnary.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taUnary.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taUnary.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taUnary.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestSDPA.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestSDPA.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestSDPA.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestSDPA.cpp From ceead52ff0b71065bdcfc74201884f333d9b68cb Mon Sep 17 00:00:00 2001 From: Stephen Jia Date: Mon, 12 Oct 2026 12:45:49 -0700 Subject: [PATCH 7/7] [ET-VK][test] Run operator benchmarks from one etvk_bench binary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pull Request resolved: https://github.com/pytorch/executorch/pull/23598 ## Ulterior Motive Make the Vulkan operator benchmarks easy to drive during optimization work: group related test cases (such as one model's bottleneck shapes) into named sets, run any subset from one binary, and later sweep implementations and dispatch parameters. ## Rationale **What**: Replace the 24 per-operator test+benchmark binaries with one entry point, `etvk_bench`, that runs test case sets registered by files under `test/ops/benchmark/cases//`. Split each operator's code into shared helpers, CPU references, and one file per set, and split each binary's default case list into correctness and performance sets. **Why**: Each binary hardcoded one mixed list of correctness-sized and performance-sized cases, so running a subset meant editing code, and modes were spread across ad hoc CLI flags, environment variables, and `#ifdef DEBUG_MODE`. ## Details - Framework - `framework/registry.{h,cpp}`: `TestCaseSet` (test cases, reference, FLOP calculator, run counts, or a custom test for checks that are not test cases) and `REGISTER_TEST_CASE_SET("", "") { ... }`. The registered body runs only when the set is selected, since building test cases may query the device. `execute_test_cases()` takes the cases directly instead of a generator. - `etvk_bench.cpp` parses gflags: `--list`, `--op=[,...]` (a parent directory such as `q8ta` selects every operator beneath it), `--set=[,...]`, `--filter=';|'` (cases whose name contains every `;`-separated substring of at least one `|`-separated alternative), `--chain=`, `--nogpu_timestamps`, `--print_output`, `--print_latencies`, `--debug`, `--device_info`, and `--helpshort`. It runs every selected set, continues past failures, and exits nonzero if any set failed. gflags comes from `external_deps` in Buck and, in the standalone CMake project, from the main ExecuTorch build tree (as the Llama runner does). - `--device-info` replaces the `test_coopmat_probe` binary (`framework/device_info.{h,cpp}`, renamed from it). - Case directories: each `cases//` has `.h` (configs and shared declarations), `.cpp` (config to `TestCase` builders, CPU references, and FLOP calculators; `q8ta/conv2d` splits these by route into `conv2d{,_dw,_pw}.cpp`), and a `test_sets/` folder whose files register the sets. A set file holds the set's test case data (configs, shapes, and the dtypes, storage types, layouts, and route selectors to sweep), the loop that turns it into test cases (including device gating and rules that derive variants from a config), and the returned `TestCaseSet`. A set gets its own `_set.cpp`; sets that share generation code share one `_sets.cpp`, named for their common prefix (`dw_sets.cpp`, `pw_sets.cpp`, `fpa_sets.cpp`, `coopmat_sets.cpp`, `scenex_sets.cpp`) or, for unprefixed sets such as `correctness` and `performance`, for the operator (`linear_sets.cpp`). Each directory's code is in its own namespace (`prototyping::`), since operators reuse names such as `LinearConfig` and now link into one binary. - Sets - Each binary's default list becomes `correctness` (its `ACCU` cases) and `performance` (its `PERF` cases), prefixed by route for secondary binaries in a directory (`dw_`, `pw_`, `fpa_`, `coopmat_`). Every case is registered in exactly one set, so the default lists no longer repeat the cases of the mode sets below. - Modes become sets: - `test_q8ta_conv2d` `--im2col-path=*`, `--narrow-workgroups-only`, `--scenex-regular`, and `--scenex-grouped` → `im2col` (each edge config under the signed, unsigned, and auto selectors), `narrow_workgroup`, `scenex_regular`, `scenex_grouped`. - Its streaming cases and dynamic shrink test → `streaming_im2col` and `streaming_dynamic_shrink`; its startup unsigned dot checks → `unsigned_dot_selection`. - `test_q8ta_conv2d_pw` `--pw-path=*` folds into `pw_correctness` and `pw_performance`, which run each config under the default, signed, unsigned, and auto selectors; `test_q8ta_conv2d_dw` narrow workgroups → `dw_narrow_workgroup`. - `SDPA_DECODE_ONLY` → `sdpa:decode` (`sdpa:performance` keeps only prefill); `COOPMAT_BENCH_CORRECTNESS_ONLY` → `q4gsw_linear:coopmat_correctness`; `SDPA_NO_CHAIN` → `--chain=1`. - `#ifdef DEBUG_MODE` and never-called easy case lists → `debug`. The `qdq` easy list now skips the legacy 4W4C path for 5D/6D shapes, as the default list does. The never-called `q8csw_conv2d` easy list is removed: it could not run, since it never set an operator name. - Dropped: per-binary banners, the coopmat benchmark's tiled-vs-coopmat summary table, and the redundant query pool initialization in `test_add`. - Build: one `etvk_bench` target in Buck and CMake replaces the per-binary targets. - CI: each Skycastle (GPU host, Eureka) and OSS workflow job keeps its name and runs `etvk_bench` with the `--op`/`--set` selection that covers its old binary's run. The default `test_q8ta_conv2d` job adds `im2col` and `narrow_workgroup`, the `_dw` job adds `dw_narrow_workgroup`, and `test_sdpa` adds `decode`; the six `--im2col-path`/`--pw-path` jobs are removed, since `test_q8ta_conv2d` and `test_q8ta_conv2d_pw` now cover them. - New files are recorded as copies of the binary they came from. CLANGTIDY findings in the moved case code are fixed (argument comments that did not match parameter names, a shadowed loop variable, narrowing, a C array). Test case construction, references, tolerances, and case names are otherwise unchanged. ghstack-source-id: 446951726 Differential Revision: [D123707389](https://our.internmc.facebook.com/intern/diff/D123707389/) --- .github/workflows/pull.yml | 17 +- .github/workflows/vulkan.yml | 20 +- .../vulkan/test/ops/benchmark/CMakeLists.txt | 52 +- .../test/ops/benchmark/cases/add/add.cpp | 60 + .../vulkan/test/ops/benchmark/cases/add/add.h | 24 + .../test/ops/benchmark/cases/add/test_add.cpp | 157 -- .../cases/add/test_sets/correctness_set.cpp | 106 ++ ...per_row.cpp => choose_qparams_per_row.cpp} | 176 +- .../choose_qparams_per_row.h | 43 + .../test_sets/choose_qparams_per_row_sets.cpp | 99 + .../conv1d/{test_conv1d_dw.cpp => conv1d.cpp} | 266 ++- .../test/ops/benchmark/cases/conv1d/conv1d.h | 59 + .../benchmark/cases/conv1d/test_conv1d_pw.cpp | 250 --- .../cases/conv1d/test_sets/dw_sets.cpp | 81 + .../cases/conv1d/test_sets/pw_sets.cpp | 85 + .../ops/benchmark/cases/conv2d/conv2d.cpp | 639 +++++++ .../test/ops/benchmark/cases/conv2d/conv2d.h | 101 ++ .../benchmark/cases/conv2d/test_conv2d.cpp | 639 ------- .../benchmark/cases/conv2d/test_conv2d_dw.cpp | 439 ----- .../benchmark/cases/conv2d/test_conv2d_pw.cpp | 267 --- .../cases/conv2d/test_sets/conv2d_sets.cpp | 342 ++++ .../cases/conv2d/test_sets/dw_sets.cpp | 189 ++ .../cases/conv2d/test_sets/pw_sets.cpp | 104 ++ .../cases/embedding_q4gsw/embedding_q4gsw.cpp | 195 ++ .../cases/embedding_q4gsw/embedding_q4gsw.h | 39 + .../embedding_q4gsw/test_embedding_q4gsw.cpp | 548 ------ .../test_sets/embedding_q4gsw_sets.cpp | 328 ++++ .../cases/mm/{test_mm.cpp => mm.cpp} | 311 +--- .../vulkan/test/ops/benchmark/cases/mm/mm.h | 49 + .../benchmark/cases/mm/test_sets/mm_sets.cpp | 171 ++ ...test_q4gsw_linear.cpp => q4gsw_linear.cpp} | 686 ++++--- .../cases/q4gsw_linear/q4gsw_linear.h | 130 ++ .../test_coopmat_linear_bench.cpp | 406 ----- .../q4gsw_linear/test_fpa_q4gsw_linear.cpp | 548 ------ .../q4gsw_linear/test_sets/coopmat_sets.cpp | 154 ++ .../q4gsw_linear/test_sets/debug_set.cpp | 58 + .../cases/q4gsw_linear/test_sets/fpa_sets.cpp | 264 +++ .../test_sets/q4gsw_linear_sets.cpp | 164 ++ ...test_q8csw_conv2d.cpp => q8csw_conv2d.cpp} | 299 +--- .../cases/q8csw_conv2d/q8csw_conv2d.h | 32 + .../test_sets/q8csw_conv2d_sets.cpp | 202 +++ ...test_q8csw_linear.cpp => q8csw_linear.cpp} | 200 +-- .../cases/q8csw_linear/q8csw_linear.h | 43 + .../q8csw_linear/test_sets/debug_set.cpp | 56 + .../test_sets/q8csw_linear_sets.cpp | 98 + .../{test_q8ta_binary.cpp => binary.cpp} | 221 +-- .../ops/benchmark/cases/q8ta/binary/binary.h | 45 + .../q8ta/binary/test_sets/binary_sets.cpp | 168 ++ .../clone/{test_q8ta_clone.cpp => clone.cpp} | 203 +-- .../ops/benchmark/cases/q8ta/clone/clone.h | 45 + .../q8ta/clone/test_sets/correctness_set.cpp | 107 ++ .../cases/q8ta/clone/test_sets/debug_set.cpp | 84 + .../benchmark/cases/q8ta/conv2d/conv2d.cpp | 605 +++++++ .../ops/benchmark/cases/q8ta/conv2d/conv2d.h | 81 + ...{test_q8ta_conv2d_dw.cpp => conv2d_dw.cpp} | 391 +--- ...{test_q8ta_conv2d_pw.cpp => conv2d_pw.cpp} | 359 +--- .../cases/q8ta/conv2d/test_q8ta_conv2d.cpp | 1594 ----------------- .../q8ta/conv2d/test_sets/conv2d_sets.cpp | 319 ++++ .../cases/q8ta/conv2d/test_sets/debug_set.cpp | 88 + .../q8ta/conv2d/test_sets/dw_debug_set.cpp | 79 + .../test_sets/dw_narrow_workgroup_set.cpp | 68 + .../cases/q8ta/conv2d/test_sets/dw_sets.cpp | 216 +++ .../q8ta/conv2d/test_sets/im2col_set.cpp | 206 +++ .../conv2d/test_sets/narrow_workgroup_set.cpp | 65 + .../cases/q8ta/conv2d/test_sets/pw_sets.cpp | 294 +++ .../q8ta/conv2d/test_sets/scenex_sets.cpp | 101 ++ .../streaming_dynamic_shrink_set.cpp | 128 ++ .../conv2d/test_sets/streaming_im2col_set.cpp | 138 ++ .../test_sets/unsigned_dot_selection_set.cpp | 43 + ...d_transposed.cpp => conv2d_transposed.cpp} | 264 +-- .../conv2d_transposed/conv2d_transposed.h | 35 + .../test_sets/conv2d_transposed_sets.cpp | 196 ++ .../{test_q8ta_linear.cpp => linear.cpp} | 144 +- .../ops/benchmark/cases/q8ta/linear/linear.h | 41 + .../q8ta/linear/test_sets/linear_sets.cpp | 96 + ...ta_pixel_shuffle.cpp => pixel_shuffle.cpp} | 150 +- .../cases/q8ta/pixel_shuffle/pixel_shuffle.h | 49 + .../test_sets/pixel_shuffle_sets.cpp | 96 + .../test/ops/benchmark/cases/q8ta/qdq/qdq.cpp | 164 ++ .../test/ops/benchmark/cases/q8ta/qdq/qdq.h | 42 + .../cases/q8ta/qdq/test_q8ta_qdq.cpp | 381 ---- .../cases/q8ta/qdq/test_sets/qdq_sets.cpp | 202 +++ .../cases/q8ta/unary/test_sets/unary_sets.cpp | 120 ++ .../unary/{test_q8ta_unary.cpp => unary.cpp} | 167 +- .../ops/benchmark/cases/q8ta/unary/unary.h | 40 + .../test/ops/benchmark/cases/sdpa/sdpa.cpp | 202 +++ .../test/ops/benchmark/cases/sdpa/sdpa.h | 50 + .../ops/benchmark/cases/sdpa/test_sdpa.cpp | 379 ---- .../cases/sdpa/test_sets/sdpa_sets.cpp | 180 ++ .../vulkan/test/ops/benchmark/etvk_bench.cpp | 247 +++ ...test_coopmat_probe.cpp => device_info.cpp} | 20 +- .../ops/benchmark/framework/device_info.h | 19 + .../test/ops/benchmark/framework/registry.cpp | 48 + .../test/ops/benchmark/framework/registry.h | 67 + .../test/ops/benchmark/framework/runner.cpp | 9 +- .../test/ops/benchmark/framework/runner.h | 4 +- .../vulkan/test/ops/benchmark/targets.bzl | 70 +- 97 files changed, 9784 insertions(+), 8542 deletions(-) create mode 100644 backends/vulkan/test/ops/benchmark/cases/add/add.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/add/add.h delete mode 100644 backends/vulkan/test/ops/benchmark/cases/add/test_add.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/add/test_sets/correctness_set.cpp rename backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/{test_choose_qparams_per_row.cpp => choose_qparams_per_row.cpp} (71%) create mode 100644 backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/choose_qparams_per_row.h create mode 100644 backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/test_sets/choose_qparams_per_row_sets.cpp rename backends/vulkan/test/ops/benchmark/cases/conv1d/{test_conv1d_dw.cpp => conv1d.cpp} (52%) create mode 100644 backends/vulkan/test/ops/benchmark/cases/conv1d/conv1d.h delete mode 100644 backends/vulkan/test/ops/benchmark/cases/conv1d/test_conv1d_pw.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/dw_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/pw_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.h delete mode 100644 backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d.cpp delete mode 100644 backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_dw.cpp delete mode 100644 backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_pw.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/conv2d_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/dw_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/pw_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.h delete mode 100644 backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_embedding_q4gsw.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_sets/embedding_q4gsw_sets.cpp rename backends/vulkan/test/ops/benchmark/cases/mm/{test_mm.cpp => mm.cpp} (68%) create mode 100644 backends/vulkan/test/ops/benchmark/cases/mm/mm.h create mode 100644 backends/vulkan/test/ops/benchmark/cases/mm/test_sets/mm_sets.cpp rename backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/{test_q4gsw_linear.cpp => q4gsw_linear.cpp} (58%) create mode 100644 backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/q4gsw_linear.h delete mode 100644 backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_coopmat_linear_bench.cpp delete mode 100644 backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_fpa_q4gsw_linear.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/coopmat_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/debug_set.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/fpa_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/q4gsw_linear_sets.cpp rename backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/{test_q8csw_conv2d.cpp => q8csw_conv2d.cpp} (74%) create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/q8csw_conv2d.h create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/test_sets/q8csw_conv2d_sets.cpp rename backends/vulkan/test/ops/benchmark/cases/q8csw_linear/{test_q8csw_linear.cpp => q8csw_linear.cpp} (75%) create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8csw_linear/q8csw_linear.h create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/debug_set.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/q8csw_linear_sets.cpp rename backends/vulkan/test/ops/benchmark/cases/q8ta/binary/{test_q8ta_binary.cpp => binary.cpp} (60%) create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/binary/binary.h create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/binary/test_sets/binary_sets.cpp rename backends/vulkan/test/ops/benchmark/cases/q8ta/clone/{test_q8ta_clone.cpp => clone.cpp} (52%) create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/clone/clone.h create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/correctness_set.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/debug_set.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.h rename backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/{test_q8ta_conv2d_dw.cpp => conv2d_dw.cpp} (62%) rename backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/{test_q8ta_conv2d_pw.cpp => conv2d_pw.cpp} (61%) delete mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/conv2d_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/debug_set.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_debug_set.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_narrow_workgroup_set.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/im2col_set.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/narrow_workgroup_set.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/pw_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/scenex_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_dynamic_shrink_set.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_im2col_set.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/unsigned_dot_selection_set.cpp rename backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/{test_q8ta_conv2d_transposed.cpp => conv2d_transposed.cpp} (71%) create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/conv2d_transposed.h create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/test_sets/conv2d_transposed_sets.cpp rename backends/vulkan/test/ops/benchmark/cases/q8ta/linear/{test_q8ta_linear.cpp => linear.cpp} (74%) create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/linear/linear.h create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/linear/test_sets/linear_sets.cpp rename backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/{test_q8ta_pixel_shuffle.cpp => pixel_shuffle.cpp} (56%) create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/pixel_shuffle.h create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/test_sets/pixel_shuffle_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.h delete mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_q8ta_qdq.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_sets/qdq_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/unary/test_sets/unary_sets.cpp rename backends/vulkan/test/ops/benchmark/cases/q8ta/unary/{test_q8ta_unary.cpp => unary.cpp} (55%) create mode 100644 backends/vulkan/test/ops/benchmark/cases/q8ta/unary/unary.h create mode 100644 backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.h delete mode 100644 backends/vulkan/test/ops/benchmark/cases/sdpa/test_sdpa.cpp create mode 100644 backends/vulkan/test/ops/benchmark/cases/sdpa/test_sets/sdpa_sets.cpp create mode 100644 backends/vulkan/test/ops/benchmark/etvk_bench.cpp rename backends/vulkan/test/ops/benchmark/framework/{test_coopmat_probe.cpp => device_info.cpp} (77%) create mode 100644 backends/vulkan/test/ops/benchmark/framework/device_info.h create mode 100644 backends/vulkan/test/ops/benchmark/framework/registry.cpp create mode 100644 backends/vulkan/test/ops/benchmark/framework/registry.h diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 925d2a6e367..6e5b0bc46a1 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -1680,14 +1680,15 @@ jobs: .ci/scripts/setup-linux.sh --build-tool "cmake" # Custom operator tests - PYTHON_EXECUTABLE=python bash backends/vulkan/test/ops/benchmark/build_and_run.sh test_add - ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8csw_linear - ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8csw_conv2d - ./cmake-out/backends/vulkan/test/ops/benchmark/test_q4gsw_linear - ./cmake-out/backends/vulkan/test/ops/benchmark/test_choose_qparams_per_row - ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8ta_qdq - ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8ta_clone - ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8ta_binary + PYTHON_EXECUTABLE=python bash backends/vulkan/test/ops/benchmark/build_and_run.sh + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=add + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8csw_linear --set=correctness,performance + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8csw_conv2d + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q4gsw_linear --set=correctness,performance + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=choose_qparams_per_row --set=correctness,performance + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8ta/qdq --set=correctness + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8ta/clone --set=correctness + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8ta/binary --set=correctness # "Classic" Operator tests PYTHON_EXECUTABLE=python bash backends/vulkan/test/scripts/test_op.sh --build diff --git a/.github/workflows/vulkan.yml b/.github/workflows/vulkan.yml index fd9919ade35..9397707d979 100644 --- a/.github/workflows/vulkan.yml +++ b/.github/workflows/vulkan.yml @@ -98,17 +98,17 @@ jobs: done # Operator coverage (mirrors test-vulkan-operators-linux, on real hardware). - # The custom-op prototyping binaries are GPU microbenchmarks that rely on - # GPU timestamp queries, so they require the hardware selected above. + # etvk_bench runs GPU microbenchmarks that rely on GPU timestamp queries, + # so it requires the hardware selected above. PYTHON_EXECUTABLE=python bash backends/vulkan/test/ops/benchmark/build_and_run.sh - ./cmake-out/backends/vulkan/test/ops/benchmark/test_add - ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8csw_linear - ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8csw_conv2d - ./cmake-out/backends/vulkan/test/ops/benchmark/test_q4gsw_linear - ./cmake-out/backends/vulkan/test/ops/benchmark/test_choose_qparams_per_row - ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8ta_qdq - ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8ta_clone - ./cmake-out/backends/vulkan/test/ops/benchmark/test_q8ta_binary + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=add + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8csw_linear --set=correctness,performance + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8csw_conv2d + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q4gsw_linear --set=correctness,performance + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=choose_qparams_per_row --set=correctness,performance + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8ta/qdq --set=correctness + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8ta/clone --set=correctness + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8ta/binary --set=correctness PYTHON_EXECUTABLE=python bash backends/vulkan/test/scripts/test_op.sh --build vulkan_sdpa_av_test diff --git a/backends/vulkan/test/ops/benchmark/CMakeLists.txt b/backends/vulkan/test/ops/benchmark/CMakeLists.txt index ff9138779ec..522d4f95f46 100644 --- a/backends/vulkan/test/ops/benchmark/CMakeLists.txt +++ b/backends/vulkan/test/ops/benchmark/CMakeLists.txt @@ -52,6 +52,8 @@ if(TARGET vulkan_backend) ${CMAKE_CURRENT_SOURCE_DIR}/framework/cm_utils.cpp ${CMAKE_CURRENT_SOURCE_DIR}/framework/config.cpp ${CMAKE_CURRENT_SOURCE_DIR}/framework/conv2d_utils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/device_info.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/registry.cpp ${CMAKE_CURRENT_SOURCE_DIR}/framework/results.cpp ${CMAKE_CURRENT_SOURCE_DIR}/framework/runner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/framework/test_case.cpp @@ -85,36 +87,24 @@ if(TARGET vulkan_backend) executorch_target_link_options_shared_lib(vulkan_backend) executorch_target_link_options_shared_lib(operator_implementations) - # Function to create operator prototype binaries - function(add_operator_prototype OPERATOR_NAME SUBDIR) - set(TARGET_NAME ${OPERATOR_NAME}) - set(SOURCE_FILE ${CMAKE_CURRENT_SOURCE_DIR}/${SUBDIR}/${OPERATOR_NAME}.cpp) - - add_executable(${TARGET_NAME} ${SOURCE_FILE} ${PROTOTYPING_UTILS_CPP}) - target_include_directories(${TARGET_NAME} PRIVATE ${COMMON_INCLUDES}) - target_link_libraries( - ${TARGET_NAME} PRIVATE vulkan_backend executorch_core - prototyping_shaderlib operator_implementations - ) - target_compile_options(${TARGET_NAME} PRIVATE ${VULKAN_CXX_FLAGS}) - set_property(TARGET ${TARGET_NAME} PROPERTY CXX_STANDARD 17) - endfunction() - - # Define operator prototypes - add_operator_prototype(test_add cases/add) - add_operator_prototype(test_q8csw_linear cases/q8csw_linear) - add_operator_prototype(test_q8csw_conv2d cases/q8csw_conv2d) - add_operator_prototype(test_q4gsw_linear cases/q4gsw_linear) - add_operator_prototype( - test_choose_qparams_per_row cases/choose_qparams_per_row + # gflags is built by the main ExecuTorch build but not installed, so find it + # in that build tree, which build_and_run.sh places five levels above this + # project's build directory. + set(gflags_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../../../third-party/gflags) + find_package(gflags REQUIRED) + + file(GLOB_RECURSE ETVK_BENCH_CASES_CPP + ${CMAKE_CURRENT_SOURCE_DIR}/cases/*.cpp + ) + add_executable( + etvk_bench ${CMAKE_CURRENT_SOURCE_DIR}/etvk_bench.cpp + ${ETVK_BENCH_CASES_CPP} ${PROTOTYPING_UTILS_CPP} + ) + target_include_directories(etvk_bench PRIVATE ${COMMON_INCLUDES}) + target_link_libraries( + etvk_bench PRIVATE vulkan_backend executorch_core prototyping_shaderlib + operator_implementations gflags ) - add_operator_prototype(test_q8ta_qdq cases/q8ta/qdq) - add_operator_prototype(test_q8ta_clone cases/q8ta/clone) - add_operator_prototype(test_q8ta_binary cases/q8ta/binary) - add_operator_prototype(test_q8ta_conv2d cases/q8ta/conv2d) - add_operator_prototype(test_q8ta_conv2d_pw cases/q8ta/conv2d) - add_operator_prototype(test_q8ta_conv2d_dw cases/q8ta/conv2d) - add_operator_prototype(test_mm cases/mm) - add_operator_prototype(test_coopmat_probe framework) - add_operator_prototype(test_coopmat_linear_bench cases/q4gsw_linear) + target_compile_options(etvk_bench PRIVATE ${VULKAN_CXX_FLAGS}) + set_property(TARGET etvk_bench PROPERTY CXX_STANDARD 17) endif() diff --git a/backends/vulkan/test/ops/benchmark/cases/add/add.cpp b/backends/vulkan/test/ops/benchmark/cases/add/add.cpp new file mode 100644 index 00000000000..1f54501ae05 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/add/add.cpp @@ -0,0 +1,60 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace add { + +// Custom FLOP calculator for add operation +// Add operation performs 1 FLOP (addition) per element +int64_t add_flop_calculator(const TestCase& test_case) { + // Calculate total elements from the first input tensor + int64_t total_elements = 1; + if (!test_case.empty() && test_case.num_inputs() > 0 && + test_case.inputs()[0].is_tensor()) { + const auto& sizes = test_case.inputs()[0].get_tensor_sizes(); + for (int64_t size : sizes) { + total_elements *= size; + } + } + + // Add operation: 1 FLOP per element (one addition) + return total_elements; +} + +// Reference implementation for add operator +void add_reference_compute(TestCase& test_case) { + const ValueSpec& input_a = test_case.inputs().at(0); + const ValueSpec& input_b = test_case.inputs().at(1); + + ValueSpec& output = test_case.outputs().at(0); + + if (input_a.dtype != vkapi::kFloat) { + throw std::invalid_argument("Unsupported dtype"); + } + + // Calculate number of elements + int64_t num_elements = input_a.numel(); + + auto& input_a_data = input_a.get_float_data(); + auto& input_b_data = input_b.get_float_data(); + + auto& ref_data = output.get_ref_float_data(); + ref_data.resize(num_elements); + for (int64_t i = 0; i < num_elements; ++i) { + ref_data[i] = input_a_data[i] + input_b_data[i]; + } +} + +} // namespace add +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/add/add.h b/backends/vulkan/test/ops/benchmark/cases/add/add.h new file mode 100644 index 00000000000..ae311090992 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/add/add.h @@ -0,0 +1,24 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace add { + +int64_t add_flop_calculator(const TestCase& test_case); + +void add_reference_compute(TestCase& test_case); + +} // namespace add +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/add/test_add.cpp b/backends/vulkan/test/ops/benchmark/cases/add/test_add.cpp deleted file mode 100644 index fed59acedf0..00000000000 --- a/backends/vulkan/test/ops/benchmark/cases/add/test_add.cpp +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include -#include -#include -#include - -using namespace executorch::vulkan::prototyping; - -// Generate test cases for add operation -std::vector generate_add_test_cases() { - std::vector test_cases; - - // Set the data generation type as a local variable - DataGenType data_gen_type = DataGenType::ONES; - - // Define different input size configurations - std::vector> size_configs = { - {1, 64, 64}, // Small square - {1, 128, 128}, // Medium square - {1, 256, 256}, // Large square - {1, 512, 512}, // Very large square - {1, 1, 1024}, // Wide tensor - {1, 1024, 1}, // Tall tensor - {32, 32, 32}, // 3D cube - {16, 128, 64}, // 3D rectangular - }; - - // Storage types to test - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - // Data types to test - std::vector data_types = {vkapi::kFloat, vkapi::kHalf}; - - // Generate test cases for each combination - for (const auto& sizes : size_configs) { - for (const auto& storage_type : storage_types) { - for (const auto& data_type : data_types) { - TestCase test_case; - - // Create a descriptive name for the test case - std::string shape_str = - shape_bracket(sizes) + "+" + shape_bracket(sizes); - std::string storage_str = repr_str(storage_type, utils::kWidthPacked); - std::string dtype_str = dtype_short(data_type); - std::string test_name = make_test_label( - "ACCU", dtype_str, dtype_str, shape_str, storage_str); - test_case.set_name(test_name); - - // Set the operator name for the test case - test_case.set_operator_name("etvk.add_prototype"); - - // Add two input tensors with the same size, type, storage, and data - // generation method - ValueSpec input_a( - sizes, data_type, storage_type, utils::kWidthPacked, data_gen_type); - ValueSpec input_b( - sizes, data_type, storage_type, utils::kWidthPacked, data_gen_type); - - // Add output tensor with the same size, type, and storage as inputs - // (output uses ZEROS by default) - ValueSpec output( - sizes, - data_type, - storage_type, - utils::kWidthPacked, - DataGenType::ZEROS); - - test_case.add_input_spec(input_a); - test_case.add_input_spec(input_b); - test_case.add_output_spec(output); - - test_cases.push_back(test_case); - } - } - } - - return test_cases; -} - -// Custom FLOP calculator for add operation -// Add operation performs 1 FLOP (addition) per element -int64_t add_flop_calculator(const TestCase& test_case) { - // Calculate total elements from the first input tensor - int64_t total_elements = 1; - if (!test_case.empty() && test_case.num_inputs() > 0 && - test_case.inputs()[0].is_tensor()) { - const auto& sizes = test_case.inputs()[0].get_tensor_sizes(); - for (int64_t size : sizes) { - total_elements *= size; - } - } - - // Add operation: 1 FLOP per element (one addition) - return total_elements; -} - -// Reference implementation for add operator -void add_reference_compute(TestCase& test_case) { - const ValueSpec& input_a = test_case.inputs().at(0); - const ValueSpec& input_b = test_case.inputs().at(1); - - ValueSpec& output = test_case.outputs().at(0); - - if (input_a.dtype != vkapi::kFloat) { - throw std::invalid_argument("Unsupported dtype"); - } - - // Calculate number of elements - int64_t num_elements = input_a.numel(); - - auto& input_a_data = input_a.get_float_data(); - auto& input_b_data = input_b.get_float_data(); - - auto& ref_data = output.get_ref_float_data(); - ref_data.resize(num_elements); - for (int64_t i = 0; i < num_elements; ++i) { - ref_data[i] = input_a_data[i] + input_b_data[i]; - } -} - -int main(int argc, char* argv[]) { - set_print_output(false); // Disable output tensor printing - set_print_latencies(false); // Enable latency timing printing - set_use_gpu_timestamps(true); // Enable GPU timestamps - - print_performance_header(); - std::cout << "Add Operation Prototyping Framework" << std::endl; - print_separator(); - - // Initialize Vulkan context - try { - api::context()->initialize_querypool(); - } catch (const std::exception& e) { - std::cerr << "Failed to initialize Vulkan context: " << e.what() - << std::endl; - return 1; - } - - // Execute test cases using the new framework with custom FLOP calculator and - // reference compute - auto results = execute_test_cases( - generate_add_test_cases, - add_flop_calculator, - "Add", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - add_reference_compute); - - return 0; -} diff --git a/backends/vulkan/test/ops/benchmark/cases/add/test_sets/correctness_set.cpp b/backends/vulkan/test/ops/benchmark/cases/add/test_sets/correctness_set.cpp new file mode 100644 index 00000000000..78da70f50b2 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/add/test_sets/correctness_set.cpp @@ -0,0 +1,106 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace add { + +namespace { + +std::vector generate_test_cases( + const std::vector>& size_configs, + const std::vector& storage_types, + const std::vector& data_types) { + std::vector test_cases; + + // Set the data generation type as a local variable + DataGenType data_gen_type = DataGenType::ONES; + + // Generate test cases for each combination + for (const auto& sizes : size_configs) { + for (const auto& storage_type : storage_types) { + for (const auto& data_type : data_types) { + TestCase test_case; + + // Create a descriptive name for the test case + std::string shape_str = + shape_bracket(sizes) + "+" + shape_bracket(sizes); + std::string storage_str = repr_str(storage_type, utils::kWidthPacked); + std::string dtype_str = dtype_short(data_type); + std::string test_name = make_test_label( + "ACCU", dtype_str, dtype_str, shape_str, storage_str); + test_case.set_name(test_name); + + // Set the operator name for the test case + test_case.set_operator_name("etvk.add_prototype"); + + // Add two input tensors with the same size, type, storage, and data + // generation method + ValueSpec input_a( + sizes, data_type, storage_type, utils::kWidthPacked, data_gen_type); + ValueSpec input_b( + sizes, data_type, storage_type, utils::kWidthPacked, data_gen_type); + + // Add output tensor with the same size, type, and storage as inputs + // (output uses ZEROS by default) + ValueSpec output( + sizes, + data_type, + storage_type, + utils::kWidthPacked, + DataGenType::ZEROS); + + test_case.add_input_spec(input_a); + test_case.add_input_spec(input_b); + test_case.add_output_spec(output); + + test_cases.push_back(test_case); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("add", "correctness") { + // Define different input size configurations + const std::vector> size_configs = { + {1, 64, 64}, // Small square + {1, 128, 128}, // Medium square + {1, 256, 256}, // Large square + {1, 512, 512}, // Very large square + {1, 1, 1024}, // Wide tensor + {1, 1024, 1}, // Tall tensor + {32, 32, 32}, // 3D cube + {16, 128, 64}, // 3D rectangular + }; + + // Storage types to test + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + + // Data types to test + const std::vector data_types = { + vkapi::kFloat, vkapi::kHalf}; + + return { + generate_test_cases(size_configs, storage_types, data_types), + add_reference_compute, + add_flop_calculator}; +} + +} // namespace add +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/test_choose_qparams_per_row.cpp b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/choose_qparams_per_row.cpp similarity index 71% rename from backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/test_choose_qparams_per_row.cpp rename to backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/choose_qparams_per_row.cpp index 48139371d89..fa96168c04a 100644 --- a/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/test_choose_qparams_per_row.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/choose_qparams_per_row.cpp @@ -4,31 +4,20 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include -#include +#include + #include #include -#include +#include +#include +#include +#include #include -#include - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 2050; -static constexpr float SMALL_SCALE_THRESHOLD = 6.1e-5f; - -// ChooseQParams configuration struct -struct ChooseQParamsConfig { - int64_t num_channels; // Height dimension (number of channels) - int64_t channel_size; // Width dimension (size per channel) - int32_t quant_min = -128; - int32_t quant_max = 127; - std::string test_case_name = "placeholder"; - std::string op_name = "choose_qparams_per_row"; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace choose_qparams_per_row { // Utility function to create a test case from a ChooseQParamsConfig TestCase create_test_case_from_config( @@ -99,6 +88,26 @@ TestCase create_test_case_from_config( return test_case; } +int64_t choose_qparams_per_channel_flop_calculator(const TestCase& test_case) { + // Get input dimensions + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + int64_t num_channels = input_sizes[0]; + int64_t channel_size = input_sizes[1]; + + // Calculate FLOPs for choose_qparams_per_channel operation + // Each channel requires: + // - Min/max finding: approximately 2 * channel_size comparisons + // - Scale calculation: ~5 operations (division, min/max operations) + // - Zero point calculation: ~10 operations (multiple arithmetic operations) + int64_t ops_per_channel = 2 * channel_size + 15; // Simplified estimate + + int64_t flop = num_channels * ops_per_channel; + + return flop; +} + +static constexpr float SMALL_SCALE_THRESHOLD = 6.1e-5f; + // CPU reference implementation matching the behavior from op_choose_qparams.cpp void calculate_scale_and_zero_point_reference( float min_val, @@ -171,90 +180,14 @@ void calculate_scale_and_zero_point_reference( } else if (initial_zero_point > qmax) { nudged_zero_point = qmax; } else { - nudged_zero_point = - static_cast(nearbyint(static_cast(initial_zero_point))); + nudged_zero_point = static_cast( + std::nearbyint(static_cast(initial_zero_point))); } scale = static_cast(scale_double); zero_point = nudged_zero_point; } -// Generate easy test cases for choose_qparams_per_channel operation (for -// debugging) -std::vector generate_choose_qparams_per_channel_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - int num_channels = 4; - int channel_size = 8; - - ChooseQParamsConfig config = { - num_channels, // num_channels - channel_size, // channel_size - -128, // quant_min - 127, // quant_max - "simple", // test_case_name - }; - - // Test with both storage types - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - std::vector float_types = {vkapi::kFloat}; - - // Generate test cases for each combination - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, input_dtype)); - } - } - - return test_cases; -} - -// Generate test cases for choose_qparams_per_channel operation -std::vector generate_choose_qparams_per_channel_test_cases() { - std::vector test_cases; - - std::vector configs = { - {4, 16}, - {8, 32}, - {16, 64}, - {32, 128}, - {64, 256}, - {128, 512}, - {1, 512}, - // Performance cases - {256, 1024}, - {512, 2048}, - {1, 2048}, - {1, 8096}, - }; - - // Test with different storage types - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - for (auto config : configs) { - std::string prefix = (config.num_channels < kRefDimSizeLimit && - config.channel_size < kRefDimSizeLimit) - ? "correctness_" - : "performance_"; - std::string generated_test_case_name = prefix + - std::to_string(config.num_channels) + "_" + - std::to_string(config.channel_size); - - config.test_case_name = generated_test_case_name; - - for (const auto& storage_type : storage_types) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, vkapi::kFloat)); - } - } - - return test_cases; -} - // Reference implementation for choose_qparams_per_channel void choose_qparams_per_channel_reference_impl(TestCase& test_case) { int32_t idx = 0; @@ -322,44 +255,7 @@ void reference_impl(TestCase& test_case) { choose_qparams_per_channel_reference_impl(test_case); } -int64_t choose_qparams_per_channel_flop_calculator(const TestCase& test_case) { - // Get input dimensions - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - int64_t num_channels = input_sizes[0]; - int64_t channel_size = input_sizes[1]; - - // Calculate FLOPs for choose_qparams_per_channel operation - // Each channel requires: - // - Min/max finding: approximately 2 * channel_size comparisons - // - Scale calculation: ~5 operations (division, min/max operations) - // - Zero point calculation: ~10 operations (multiple arithmetic operations) - int64_t ops_per_channel = 2 * channel_size + 15; // Simplified estimate - - int64_t flop = num_channels * ops_per_channel; - - return flop; -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Choose QParams Per Channel Operation Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_choose_qparams_per_channel_test_cases, - choose_qparams_per_channel_flop_calculator, - "ChooseQParamsPerChannel", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace choose_qparams_per_row +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/choose_qparams_per_row.h b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/choose_qparams_per_row.h new file mode 100644 index 00000000000..88ce7f05028 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/choose_qparams_per_row.h @@ -0,0 +1,43 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace choose_qparams_per_row { + +constexpr int64_t kRefDimSizeLimit = 2050; + +// ChooseQParams configuration struct +struct ChooseQParamsConfig { + int64_t num_channels; // Height dimension (number of channels) + int64_t channel_size; // Width dimension (size per channel) + int32_t quant_min = -128; + int32_t quant_max = 127; + std::string test_case_name = "placeholder"; + std::string op_name = "choose_qparams_per_row"; +}; + +TestCase create_test_case_from_config( + const ChooseQParamsConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype); + +void reference_impl(TestCase& test_case); + +int64_t choose_qparams_per_channel_flop_calculator(const TestCase& test_case); + +} // namespace choose_qparams_per_row +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/test_sets/choose_qparams_per_row_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/test_sets/choose_qparams_per_row_sets.cpp new file mode 100644 index 00000000000..6b95e4ff303 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/test_sets/choose_qparams_per_row_sets.cpp @@ -0,0 +1,99 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace choose_qparams_per_row { + +namespace { + +std::vector generate_test_cases( + const std::vector& configs, + const std::vector& storage_types) { + std::vector test_cases; + + for (auto config : configs) { + std::string prefix = (config.num_channels < kRefDimSizeLimit && + config.channel_size < kRefDimSizeLimit) + ? "correctness_" + : "performance_"; + std::string generated_test_case_name = prefix + + std::to_string(config.num_channels) + "_" + + std::to_string(config.channel_size); + + config.test_case_name = generated_test_case_name; + + for (const auto& storage_type : storage_types) { + test_cases.push_back( + create_test_case_from_config(config, storage_type, vkapi::kFloat)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("choose_qparams_per_row", "correctness") { + const std::vector configs = { + {4, 16}, + {8, 32}, + {16, 64}, + {32, 128}, + {64, 256}, + {128, 512}, + {1, 512}, + {256, 1024}, + {512, 2048}, + {1, 2048}, + }; + // Test with different storage types + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + return { + generate_test_cases(configs, storage_types), + reference_impl, + choose_qparams_per_channel_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("choose_qparams_per_row", "debug") { + const std::vector configs = { + // Single simple configuration for debugging + {4, 8}, + }; + // Test with different storage types + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + return { + generate_test_cases(configs, storage_types), + reference_impl, + choose_qparams_per_channel_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("choose_qparams_per_row", "performance") { + const std::vector configs = { + // Performance cases + {1, 8096}, + }; + // Test with different storage types + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + return { + generate_test_cases(configs, storage_types), + reference_impl, + choose_qparams_per_channel_flop_calculator}; +} + +} // namespace choose_qparams_per_row +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv1d/test_conv1d_dw.cpp b/backends/vulkan/test/ops/benchmark/cases/conv1d/conv1d.cpp similarity index 52% rename from backends/vulkan/test/ops/benchmark/cases/conv1d/test_conv1d_dw.cpp rename to backends/vulkan/test/ops/benchmark/cases/conv1d/conv1d.cpp index 8e1234611c9..136f9fa6cf2 100644 --- a/backends/vulkan/test/ops/benchmark/cases/conv1d/test_conv1d_dw.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/conv1d/conv1d.cpp @@ -4,31 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include - -#include -#include - -#include +#include -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 256; +#include +#include +#include -struct Conv1dDWConfig { - int64_t N; - int64_t C; - int64_t L; - int64_t K; - int64_t stride; - int64_t padding; - int64_t dilation; - bool has_bias; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv1d { -static TestCase create_conv1d_dw_test_case( +TestCase create_conv1d_dw_test_case( const Conv1dDWConfig& config, vkapi::ScalarType dtype, utils::StorageType storage_type) { @@ -126,7 +113,123 @@ static TestCase create_conv1d_dw_test_case( return test_case; } -static void conv1d_dw_reference_impl(TestCase& test_case) { +TestCase create_conv1d_pw_test_case( + const Conv1dPWConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type) { + TestCase test_case; + + bool is_perf = config.C_in > kRefDimSizeLimit || + config.C_out > kRefDimSizeLimit || config.L > kRefDimSizeLimit; + + std::string prefix = is_perf ? "PERF" : "ACCU"; + std::string storage_str = storage_type_abbrev(storage_type) + "(HP)"; + std::string dtype_str = dtype_short(dtype); + + std::string bias_str = config.has_bias ? "+bias" : ""; + + std::string shape = "[" + std::to_string(config.N) + "," + + std::to_string(config.C_in) + "," + std::to_string(config.L) + "]x[" + + std::to_string(config.C_out) + "," + std::to_string(config.C_in) + ",1]"; + + std::string name = make_test_label( + prefix, dtype_str, dtype_str, shape, storage_str, bias_str); + + test_case.set_name(name); + test_case.set_operator_name("test_etvk.test_conv1d_pw.default"); + + // Input: [N, C_in, L] height-packed + ValueSpec input( + {config.N, config.C_in, config.L}, + dtype, + storage_type, + utils::kHeightPacked, + DataGenType::RANDOM); + test_case.add_input_spec(input); + + // Weight: [C_out, C_in, 1] height-packed, constant + ValueSpec weight( + {config.C_out, config.C_in, 1}, + dtype, + storage_type, + utils::kHeightPacked, + DataGenType::RANDOM); + weight.set_constant(true); + test_case.add_input_spec(weight); + + // Bias: [C_out] or None + if (config.has_bias) { + ValueSpec bias( + {config.C_out}, + dtype, + storage_type, + utils::kWidthPacked, + DataGenType::RANDOM); + bias.set_constant(true); + test_case.add_input_spec(bias); + } else { + ValueSpec none_bias(static_cast(0)); + none_bias.set_none(true); + test_case.add_input_spec(none_bias); + } + + // stride = [1] + test_case.add_input_spec(ValueSpec(std::vector{1})); + // padding = [0] + test_case.add_input_spec(ValueSpec(std::vector{0})); + // dilation = [1] + test_case.add_input_spec(ValueSpec(std::vector{1})); + // groups = 1 + test_case.add_input_spec(ValueSpec(static_cast(1))); + + // Output: [N, C_out, L] height-packed + ValueSpec output( + {config.N, config.C_out, config.L}, + dtype, + storage_type, + utils::kHeightPacked, + DataGenType::ZEROS); + test_case.add_output_spec(output); + + if (dtype == vkapi::kHalf) { + test_case.set_abs_tolerance(1e-1f); + test_case.set_rel_tolerance(1e-2f); + } else { + test_case.set_abs_tolerance(1e-3f); + test_case.set_rel_tolerance(1e-3f); + } + + test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); + + return test_case; +} + +int64_t conv1d_dw_flop_calculator(const TestCase& test_case) { + auto in_sizes = test_case.inputs()[0].get_tensor_sizes(); + auto w_sizes = test_case.inputs()[1].get_tensor_sizes(); + auto out_sizes = test_case.outputs()[0].get_tensor_sizes(); + + const int64_t N = in_sizes[0]; + const int64_t C = in_sizes[1]; + const int64_t K = w_sizes[2]; + const int64_t L_out = out_sizes[2]; + + return 2 * N * C * L_out * K; +} + +int64_t conv1d_pw_flop_calculator(const TestCase& test_case) { + auto in_sizes = test_case.inputs()[0].get_tensor_sizes(); + auto w_sizes = test_case.inputs()[1].get_tensor_sizes(); + + int64_t N = in_sizes[0]; + int64_t C_in = in_sizes[1]; + int64_t L = in_sizes[2]; + int64_t C_out = w_sizes[0]; + + return 2 * N * C_in * C_out * L; +} + +void conv1d_dw_reference_impl(TestCase& test_case) { const auto& input_spec = test_case.inputs()[0]; const auto& weight_spec = test_case.inputs()[1]; const auto& bias_spec = test_case.inputs()[2]; @@ -185,83 +288,56 @@ static void conv1d_dw_reference_impl(TestCase& test_case) { } } -static std::vector generate_conv1d_dw_test_cases() { - std::vector test_cases; - - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - // Accuracy shapes - std::vector accu_configs = { - // {N, C, L, K, stride, padding, dilation, has_bias} - {1, 16, 64, 3, 1, 1, 1, false}, - {1, 32, 128, 5, 1, 2, 1, true}, - {1, 64, 32, 3, 2, 1, 1, false}, - {2, 16, 64, 3, 1, 1, 1, true}, - {1, 16, 64, 7, 1, 3, 2, false}, - // Non-aligned channel counts (not a multiple of 4) - {1, 5, 64, 3, 1, 1, 1, false}, - {1, 5, 64, 3, 1, 1, 1, true}, - {1, 7, 32, 5, 1, 2, 1, false}, - {1, 13, 48, 3, 2, 1, 1, true}, - {2, 7, 64, 3, 1, 1, 1, false}, - }; - - for (const auto& cfg : accu_configs) { - for (auto st : storage_types) { - test_cases.push_back(create_conv1d_dw_test_case(cfg, vkapi::kFloat, st)); - } - } +void conv1d_pw_reference_impl(TestCase& test_case) { + const auto& input_spec = test_case.inputs()[0]; + const auto& weight_spec = test_case.inputs()[1]; + const auto& bias_spec = test_case.inputs()[2]; + ValueSpec& output = test_case.outputs()[0]; - // Performance shapes (half + float) - std::vector perf_configs = { - {1, 256, 1024, 3, 1, 1, 1, false}, - {1, 512, 2048, 5, 1, 2, 1, true}, - {1, 128, 4096, 31, 1, 15, 1, false}, - }; - - for (const auto& cfg : perf_configs) { - for (auto st : storage_types) { - test_cases.push_back(create_conv1d_dw_test_case(cfg, vkapi::kFloat, st)); - test_cases.push_back(create_conv1d_dw_test_case(cfg, vkapi::kHalf, st)); - } + if (input_spec.dtype != vkapi::kFloat) { + throw std::invalid_argument("Reference only supports float"); } - return test_cases; -} - -static int64_t conv1d_dw_flop_calculator(const TestCase& test_case) { - auto in_sizes = test_case.inputs()[0].get_tensor_sizes(); - auto w_sizes = test_case.inputs()[1].get_tensor_sizes(); - auto out_sizes = test_case.outputs()[0].get_tensor_sizes(); - - const int64_t N = in_sizes[0]; - const int64_t C = in_sizes[1]; - const int64_t K = w_sizes[2]; - const int64_t L_out = out_sizes[2]; - - return 2 * N * C * L_out * K; -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); + auto in_sizes = input_spec.get_tensor_sizes(); + auto w_sizes = weight_spec.get_tensor_sizes(); - print_performance_header(); - std::cout << "Conv1d Depthwise (Height-Packed) Benchmark" << std::endl; - print_separator(); + int64_t N = in_sizes[0]; + int64_t C_in = in_sizes[1]; + int64_t L = in_sizes[2]; + int64_t C_out = w_sizes[0]; - ReferenceComputeFunc ref_fn = conv1d_dw_reference_impl; + const auto& in_data = input_spec.get_float_data(); + const auto& w_data = weight_spec.get_float_data(); + auto& ref_data = output.get_ref_float_data(); + ref_data.resize(N * C_out * L, 0.0f); - auto results = execute_test_cases( - generate_conv1d_dw_test_cases, - conv1d_dw_flop_calculator, - "Conv1dDW", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); + // input is NCHW-contiguous: [N, C_in, L] + // weight is [C_out, C_in, 1] + for (int64_t n = 0; n < N; ++n) { + for (int64_t oc = 0; oc < C_out; ++oc) { + for (int64_t l = 0; l < L; ++l) { + float sum = 0.0f; + for (int64_t ic = 0; ic < C_in; ++ic) { + sum += in_data[n * C_in * L + ic * L + l] * w_data[oc * C_in + ic]; + } + ref_data[n * C_out * L + oc * L + l] = sum; + } + } + } - return 0; + if (!bias_spec.is_none()) { + const auto& bias_data = bias_spec.get_float_data(); + for (int64_t n = 0; n < N; ++n) { + for (int64_t oc = 0; oc < C_out; ++oc) { + for (int64_t l = 0; l < L; ++l) { + ref_data[n * C_out * L + oc * L + l] += bias_data[oc]; + } + } + } + } } + +} // namespace conv1d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv1d/conv1d.h b/backends/vulkan/test/ops/benchmark/cases/conv1d/conv1d.h new file mode 100644 index 00000000000..c915e9c91d9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv1d/conv1d.h @@ -0,0 +1,59 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv1d { + +constexpr int64_t kRefDimSizeLimit = 256; + +struct Conv1dDWConfig { + int64_t N; + int64_t C; + int64_t L; + int64_t K; + int64_t stride; + int64_t padding; + int64_t dilation; + bool has_bias; +}; + +struct Conv1dPWConfig { + int64_t N; + int64_t C_in; + int64_t C_out; + int64_t L; + bool has_bias; +}; + +TestCase create_conv1d_dw_test_case( + const Conv1dDWConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type); + +TestCase create_conv1d_pw_test_case( + const Conv1dPWConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type); + +void conv1d_dw_reference_impl(TestCase& test_case); + +void conv1d_pw_reference_impl(TestCase& test_case); + +int64_t conv1d_dw_flop_calculator(const TestCase& test_case); + +int64_t conv1d_pw_flop_calculator(const TestCase& test_case); + +} // namespace conv1d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv1d/test_conv1d_pw.cpp b/backends/vulkan/test/ops/benchmark/cases/conv1d/test_conv1d_pw.cpp deleted file mode 100644 index aab1862c75d..00000000000 --- a/backends/vulkan/test/ops/benchmark/cases/conv1d/test_conv1d_pw.cpp +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include - -#include -#include - -#include - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 256; - -struct Conv1dPWConfig { - int64_t N; - int64_t C_in; - int64_t C_out; - int64_t L; - bool has_bias; -}; - -static TestCase create_conv1d_pw_test_case( - const Conv1dPWConfig& config, - vkapi::ScalarType dtype, - utils::StorageType storage_type) { - TestCase test_case; - - bool is_perf = config.C_in > kRefDimSizeLimit || - config.C_out > kRefDimSizeLimit || config.L > kRefDimSizeLimit; - - std::string prefix = is_perf ? "PERF" : "ACCU"; - std::string storage_str = storage_type_abbrev(storage_type) + "(HP)"; - std::string dtype_str = dtype_short(dtype); - - std::string bias_str = config.has_bias ? "+bias" : ""; - - std::string shape = "[" + std::to_string(config.N) + "," + - std::to_string(config.C_in) + "," + std::to_string(config.L) + "]x[" + - std::to_string(config.C_out) + "," + std::to_string(config.C_in) + ",1]"; - - std::string name = make_test_label( - prefix, dtype_str, dtype_str, shape, storage_str, bias_str); - - test_case.set_name(name); - test_case.set_operator_name("test_etvk.test_conv1d_pw.default"); - - // Input: [N, C_in, L] height-packed - ValueSpec input( - {config.N, config.C_in, config.L}, - dtype, - storage_type, - utils::kHeightPacked, - DataGenType::RANDOM); - test_case.add_input_spec(input); - - // Weight: [C_out, C_in, 1] height-packed, constant - ValueSpec weight( - {config.C_out, config.C_in, 1}, - dtype, - storage_type, - utils::kHeightPacked, - DataGenType::RANDOM); - weight.set_constant(true); - test_case.add_input_spec(weight); - - // Bias: [C_out] or None - if (config.has_bias) { - ValueSpec bias( - {config.C_out}, - dtype, - storage_type, - utils::kWidthPacked, - DataGenType::RANDOM); - bias.set_constant(true); - test_case.add_input_spec(bias); - } else { - ValueSpec none_bias(static_cast(0)); - none_bias.set_none(true); - test_case.add_input_spec(none_bias); - } - - // stride = [1] - test_case.add_input_spec(ValueSpec(std::vector{1})); - // padding = [0] - test_case.add_input_spec(ValueSpec(std::vector{0})); - // dilation = [1] - test_case.add_input_spec(ValueSpec(std::vector{1})); - // groups = 1 - test_case.add_input_spec(ValueSpec(static_cast(1))); - - // Output: [N, C_out, L] height-packed - ValueSpec output( - {config.N, config.C_out, config.L}, - dtype, - storage_type, - utils::kHeightPacked, - DataGenType::ZEROS); - test_case.add_output_spec(output); - - if (dtype == vkapi::kHalf) { - test_case.set_abs_tolerance(1e-1f); - test_case.set_rel_tolerance(1e-2f); - } else { - test_case.set_abs_tolerance(1e-3f); - test_case.set_rel_tolerance(1e-3f); - } - - test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); - - return test_case; -} - -static void conv1d_pw_reference_impl(TestCase& test_case) { - const auto& input_spec = test_case.inputs()[0]; - const auto& weight_spec = test_case.inputs()[1]; - const auto& bias_spec = test_case.inputs()[2]; - ValueSpec& output = test_case.outputs()[0]; - - if (input_spec.dtype != vkapi::kFloat) { - throw std::invalid_argument("Reference only supports float"); - } - - auto in_sizes = input_spec.get_tensor_sizes(); - auto w_sizes = weight_spec.get_tensor_sizes(); - - int64_t N = in_sizes[0]; - int64_t C_in = in_sizes[1]; - int64_t L = in_sizes[2]; - int64_t C_out = w_sizes[0]; - - const auto& in_data = input_spec.get_float_data(); - const auto& w_data = weight_spec.get_float_data(); - auto& ref_data = output.get_ref_float_data(); - ref_data.resize(N * C_out * L, 0.0f); - - // input is NCHW-contiguous: [N, C_in, L] - // weight is [C_out, C_in, 1] - for (int64_t n = 0; n < N; ++n) { - for (int64_t oc = 0; oc < C_out; ++oc) { - for (int64_t l = 0; l < L; ++l) { - float sum = 0.0f; - for (int64_t ic = 0; ic < C_in; ++ic) { - sum += in_data[n * C_in * L + ic * L + l] * w_data[oc * C_in + ic]; - } - ref_data[n * C_out * L + oc * L + l] = sum; - } - } - } - - if (!bias_spec.is_none()) { - const auto& bias_data = bias_spec.get_float_data(); - for (int64_t n = 0; n < N; ++n) { - for (int64_t oc = 0; oc < C_out; ++oc) { - for (int64_t l = 0; l < L; ++l) { - ref_data[n * C_out * L + oc * L + l] += bias_data[oc]; - } - } - } - } -} - -static std::vector generate_conv1d_pw_test_cases() { - std::vector test_cases; - - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - // Accuracy shapes (float, small) - std::vector accu_configs = { - {1, 16, 32, 64, false}, - {1, 16, 32, 64, true}, - {1, 32, 16, 128, false}, - {1, 32, 16, 128, true}, - {1, 64, 64, 32, false}, - {1, 128, 256, 16, true}, - {2, 16, 32, 64, false}, - {2, 16, 32, 64, true}, - // Non-aligned channel counts (not a multiple of 4) - {1, 5, 7, 64, false}, - {1, 5, 7, 64, true}, - {1, 13, 17, 48, false}, - {1, 13, 17, 48, true}, - {1, 7, 5, 32, false}, - {2, 5, 13, 64, true}, - }; - - for (const auto& cfg : accu_configs) { - for (auto st : storage_types) { - test_cases.push_back(create_conv1d_pw_test_case(cfg, vkapi::kFloat, st)); - } - } - - // Performance shapes (half + float) - std::vector perf_configs = { - {1, 256, 512, 1024, false}, - {1, 256, 512, 1024, true}, - {1, 512, 256, 2048, false}, - {1, 128, 128, 4096, true}, - }; - - for (const auto& cfg : perf_configs) { - for (auto st : storage_types) { - test_cases.push_back(create_conv1d_pw_test_case(cfg, vkapi::kFloat, st)); - test_cases.push_back(create_conv1d_pw_test_case(cfg, vkapi::kHalf, st)); - } - } - - return test_cases; -} - -static int64_t conv1d_pw_flop_calculator(const TestCase& test_case) { - auto in_sizes = test_case.inputs()[0].get_tensor_sizes(); - auto w_sizes = test_case.inputs()[1].get_tensor_sizes(); - - int64_t N = in_sizes[0]; - int64_t C_in = in_sizes[1]; - int64_t L = in_sizes[2]; - int64_t C_out = w_sizes[0]; - - return 2 * N * C_in * C_out * L; -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Conv1d Pointwise (Height-Packed) Benchmark" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = conv1d_pw_reference_impl; - - auto results = execute_test_cases( - generate_conv1d_pw_test_cases, - conv1d_pw_flop_calculator, - "Conv1dPW", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/dw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/dw_sets.cpp new file mode 100644 index 00000000000..24ed642e6aa --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/dw_sets.cpp @@ -0,0 +1,81 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv1d { + +namespace { + +std::vector generate_dw_test_cases( + const std::vector& configs, + const std::vector& storage_types, + const std::vector& dtypes) { + std::vector test_cases; + + for (const auto& cfg : configs) { + for (auto st : storage_types) { + for (auto dtype : dtypes) { + test_cases.push_back(create_conv1d_dw_test_case(cfg, dtype, st)); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("conv1d", "dw_correctness") { + // Accuracy shapes + const std::vector configs = { + // {N, C, L, K, stride, padding, dilation, has_bias} + {1, 16, 64, 3, 1, 1, 1, false}, + {1, 32, 128, 5, 1, 2, 1, true}, + {1, 64, 32, 3, 2, 1, 1, false}, + {2, 16, 64, 3, 1, 1, 1, true}, + {1, 16, 64, 7, 1, 3, 2, false}, + // Non-aligned channel counts (not a multiple of 4) + {1, 5, 64, 3, 1, 1, 1, false}, + {1, 5, 64, 3, 1, 1, 1, true}, + {1, 7, 32, 5, 1, 2, 1, false}, + {1, 13, 48, 3, 2, 1, 1, true}, + {2, 7, 64, 3, 1, 1, 1, false}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector dtypes = {vkapi::kFloat}; + return { + generate_dw_test_cases(configs, storage_types, dtypes), + conv1d_dw_reference_impl, + conv1d_dw_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("conv1d", "dw_performance") { + // Performance shapes (half + float) + const std::vector configs = { + {1, 256, 1024, 3, 1, 1, 1, false}, + {1, 512, 2048, 5, 1, 2, 1, true}, + {1, 128, 4096, 31, 1, 15, 1, false}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; + return { + generate_dw_test_cases(configs, storage_types, dtypes), + conv1d_dw_reference_impl, + conv1d_dw_flop_calculator}; +} + +} // namespace conv1d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/pw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/pw_sets.cpp new file mode 100644 index 00000000000..67f6b873425 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/pw_sets.cpp @@ -0,0 +1,85 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv1d { + +namespace { + +std::vector generate_pw_test_cases( + const std::vector& configs, + const std::vector& storage_types, + const std::vector& dtypes) { + std::vector test_cases; + + for (const auto& cfg : configs) { + for (auto st : storage_types) { + for (auto dtype : dtypes) { + test_cases.push_back(create_conv1d_pw_test_case(cfg, dtype, st)); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("conv1d", "pw_correctness") { + // Accuracy shapes (float, small) + const std::vector configs = { + {1, 16, 32, 64, false}, + {1, 16, 32, 64, true}, + {1, 32, 16, 128, false}, + {1, 32, 16, 128, true}, + {1, 64, 64, 32, false}, + {1, 128, 256, 16, true}, + {2, 16, 32, 64, false}, + {2, 16, 32, 64, true}, + // Non-aligned channel counts (not a multiple of 4) + {1, 5, 7, 64, false}, + {1, 5, 7, 64, true}, + {1, 13, 17, 48, false}, + {1, 13, 17, 48, true}, + {1, 7, 5, 32, false}, + {2, 5, 13, 64, true}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector dtypes = {vkapi::kFloat}; + return { + generate_pw_test_cases(configs, storage_types, dtypes), + conv1d_pw_reference_impl, + conv1d_pw_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("conv1d", "pw_performance") { + // Performance shapes (half + float) + const std::vector configs = { + {1, 256, 512, 1024, false}, + {1, 256, 512, 1024, true}, + {1, 512, 256, 2048, false}, + {1, 128, 128, 4096, true}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; + return { + generate_pw_test_cases(configs, storage_types, dtypes), + conv1d_pw_reference_impl, + conv1d_pw_flop_calculator}; +} + +} // namespace conv1d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.cpp new file mode 100644 index 00000000000..8421257c40b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.cpp @@ -0,0 +1,639 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv2d { + +namespace { + +int64_t calc_out_size( + int64_t in_size, + int64_t kernel_size, + int64_t stride, + int64_t padding, + int64_t dilation) { + return (in_size + 2 * padding - dilation * (kernel_size - 1) - 1) / stride + + 1; +} + +} // namespace + +// Shared perf/skip classification used by both create_conv2d_test_case (to tag +// PERF vs ACCU) and conv2d_reference_impl (to gate the large-K FP16 reference +// check). A shape is "perf" if any dimension reaches kRefDimSizeLimit; the +// boundary is inclusive (>=) so a 64-wide dim counts as perf — FP16 +// accumulation error at K = K_h * K_w * C_in for such shapes can exceed the +// half tolerance and false-fail. Keep both call sites on this single helper to +// avoid the two predicates drifting apart. +bool conv2d_is_perf_shape(int64_t C_in, int64_t C_out, int64_t H, int64_t W) { + return C_in >= kRefDimSizeLimit || C_out >= kRefDimSizeLimit || + H >= kRefDimSizeLimit || W >= kRefDimSizeLimit; +} + +TestCase create_conv2d_test_case( + const Conv2dTestConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout, + const std::string& impl_selector) { + TestCase test_case; + + bool is_perf = conv2d_is_perf_shape( + config.dims.C, config.C_out, config.dims.H, config.dims.W); + + std::string prefix = is_perf ? "PERF" : "ACCU"; + std::string storage_str = repr_str(storage_type, memory_layout); + std::string dtype_str = dtype_short(dtype); + std::string bias_str = config.has_bias ? "+bias" : ""; + + int64_t H_out = calc_out_size( + config.dims.H, + config.kernel.h, + config.stride.h, + config.padding.h, + config.dilation.h); + int64_t W_out = calc_out_size( + config.dims.W, + config.kernel.w, + config.stride.w, + config.padding.w, + config.dilation.w); + + std::string shape = "[" + std::to_string(config.dims.N) + "," + + std::to_string(config.dims.C) + "," + std::to_string(config.dims.H) + + "," + std::to_string(config.dims.W) + "]->[" + + std::to_string(config.C_out) + "] k" + std::to_string(config.kernel.h) + + "x" + std::to_string(config.kernel.w) + " s" + + std::to_string(config.stride.h) + " p" + + std::to_string(config.padding.h) + " d" + + std::to_string(config.dilation.h); + + std::string suffix = bias_str; + if (!impl_selector.empty()) { + if (!suffix.empty()) { + suffix += " "; + } + suffix += "[" + impl_selector + "]"; + } + + std::string name = + make_test_label(prefix, dtype_str, dtype_str, shape, storage_str, suffix); + + test_case.set_name(name); + test_case.set_operator_name("test_etvk.test_conv2d.default"); + + // Input tensor [N, C_in, H, W] + ValueSpec input( + {config.dims.N, config.dims.C, config.dims.H, config.dims.W}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + + // Weight tensor [C_out, C_in, K_h, K_w] - constant + ValueSpec weight( + {config.C_out, config.dims.C, config.kernel.h, config.kernel.w}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + weight.set_constant(true); + + test_case.add_input_spec(input); + test_case.add_input_spec(weight); + + // Bias (or none) + if (config.has_bias) { + ValueSpec bias( + {config.C_out}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + bias.set_constant(true); + test_case.add_input_spec(bias); + } else { + ValueSpec none_bias(static_cast(0)); + none_bias.set_none(true); + test_case.add_input_spec(none_bias); + } + + // stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w + test_case.add_input_spec(ValueSpec(static_cast(config.stride.h))); + test_case.add_input_spec(ValueSpec(static_cast(config.stride.w))); + test_case.add_input_spec(ValueSpec(static_cast(config.padding.h))); + test_case.add_input_spec(ValueSpec(static_cast(config.padding.w))); + test_case.add_input_spec(ValueSpec(static_cast(config.dilation.h))); + test_case.add_input_spec(ValueSpec(static_cast(config.dilation.w))); + + // impl_selector string + test_case.add_input_spec(ValueSpec::make_string(impl_selector)); + + // Output tensor [N, C_out, H_out, W_out] + ValueSpec output( + {config.dims.N, config.C_out, H_out, W_out}, + dtype, + storage_type, + memory_layout, + DataGenType::ZEROS); + test_case.add_output_spec(output); + + if (dtype == vkapi::kHalf) { + test_case.set_abs_tolerance(1e-1f); + test_case.set_rel_tolerance(1e-2f); + } else { + test_case.set_abs_tolerance(1e-3f); + test_case.set_rel_tolerance(1e-3f); + } + + test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); + + return test_case; +} + +TestCase create_conv2d_dw_test_case( + const Conv2dDwConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout, + const std::string& impl_selector) { + TestCase test_case; + + bool is_perf = config.dims.C > kRefDimSizeLimit || + config.dims.H > kRefDimSizeLimit || config.dims.W > kRefDimSizeLimit; + + std::string prefix = is_perf ? "PERF" : "ACCU"; + std::string storage_str = repr_str(storage_type, memory_layout); + std::string dtype_str = dtype_short(dtype); + std::string bias_str = config.has_bias ? "+bias" : ""; + + int64_t H_out = calc_out_size( + config.dims.H, + config.kernel.h, + config.stride.h, + config.padding.h, + config.dilation.h); + int64_t W_out = calc_out_size( + config.dims.W, + config.kernel.w, + config.stride.w, + config.padding.w, + config.dilation.w); + (void)H_out; + (void)W_out; + + // groups for depthwise conv2d == number of input channels + std::string shape = "[" + std::to_string(config.dims.N) + "," + + std::to_string(config.dims.C) + "," + std::to_string(config.dims.H) + + "," + std::to_string(config.dims.W) + "] k" + + std::to_string(config.kernel.h) + " s" + std::to_string(config.stride.h) + + " p" + std::to_string(config.padding.h) + " d" + + std::to_string(config.dilation.h) + " g" + std::to_string(config.dims.C); + + std::string suffix = bias_str; + if (!impl_selector.empty()) { + if (!suffix.empty()) { + suffix += " "; + } + suffix += "[" + impl_selector + "]"; + } + + std::string name = + make_test_label(prefix, dtype_str, dtype_str, shape, storage_str, suffix); + + test_case.set_name(name); + test_case.set_operator_name("test_etvk.test_conv2d_dw.default"); + + // Input tensor [N, C, H, W] + ValueSpec input( + {config.dims.N, config.dims.C, config.dims.H, config.dims.W}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + + // Weight tensor [C, 1, K_h, K_w] - constant + ValueSpec weight( + {config.dims.C, 1, config.kernel.h, config.kernel.w}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + weight.set_constant(true); + + test_case.add_input_spec(input); + test_case.add_input_spec(weight); + + // Bias (or none) + if (config.has_bias) { + ValueSpec bias( + {config.dims.C}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + bias.set_constant(true); + test_case.add_input_spec(bias); + } else { + ValueSpec none_bias(static_cast(0)); + none_bias.set_none(true); + test_case.add_input_spec(none_bias); + } + + // stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w + test_case.add_input_spec(ValueSpec(static_cast(config.stride.h))); + test_case.add_input_spec(ValueSpec(static_cast(config.stride.w))); + test_case.add_input_spec(ValueSpec(static_cast(config.padding.h))); + test_case.add_input_spec(ValueSpec(static_cast(config.padding.w))); + test_case.add_input_spec(ValueSpec(static_cast(config.dilation.h))); + test_case.add_input_spec(ValueSpec(static_cast(config.dilation.w))); + + // impl_selector string + test_case.add_input_spec(ValueSpec::make_string(impl_selector)); + + // Output tensor [N, C, H_out, W_out] + ValueSpec output( + {config.dims.N, config.dims.C, H_out, W_out}, + dtype, + storage_type, + memory_layout, + DataGenType::ZEROS); + test_case.add_output_spec(output); + + if (dtype == vkapi::kHalf) { + test_case.set_abs_tolerance(1e-1f); + test_case.set_rel_tolerance(1e-2f); + } else { + test_case.set_abs_tolerance(1e-3f); + test_case.set_rel_tolerance(1e-3f); + } + + test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); + + return test_case; +} + +TestCase create_conv2d_pw_test_case( + const Conv2dPwConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout) { + TestCase test_case; + + bool is_perf = config.C_in > kRefDimSizeLimit || + config.C_out > kRefDimSizeLimit || config.H > kRefDimSizeLimit || + config.W > kRefDimSizeLimit; + + std::string prefix = is_perf ? "PERF" : "ACCU"; + std::string storage_str = repr_str(storage_type, memory_layout); + std::string dtype_str = dtype_short(dtype); + std::string bias_str = config.has_bias ? "+bias" : ""; + + // Pointwise conv2d: kernel 1x1, stride 1, pad 0, dilation 1, groups 1 + std::string shape = "[" + std::to_string(config.N) + "," + + std::to_string(config.C_in) + "," + std::to_string(config.H) + "," + + std::to_string(config.W) + "]x[" + std::to_string(config.C_out) + "," + + std::to_string(config.C_in) + ",1,1] s1 p0 d1 g1"; + + std::string name = make_test_label( + prefix, dtype_str, dtype_str, shape, storage_str, bias_str); + + test_case.set_name(name); + test_case.set_operator_name("test_etvk.test_conv2d_pw.default"); + + // Input tensor [N, C_in, H, W] + ValueSpec input( + {config.N, config.C_in, config.H, config.W}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + + // Weight tensor [C_out, C_in, 1, 1] - constant + ValueSpec weight( + {config.C_out, config.C_in, 1, 1}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + weight.set_constant(true); + + test_case.add_input_spec(input); + test_case.add_input_spec(weight); + + // Bias (or none) + if (config.has_bias) { + ValueSpec bias( + {config.C_out}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + bias.set_constant(true); + test_case.add_input_spec(bias); + } else { + ValueSpec none_bias(static_cast(0)); + none_bias.set_none(true); + test_case.add_input_spec(none_bias); + } + + // impl_selector + ValueSpec impl_selector_spec = ValueSpec::make_string("default"); + test_case.add_input_spec(impl_selector_spec); + + // Output tensor [N, C_out, H, W] + ValueSpec output( + {config.N, config.C_out, config.H, config.W}, + dtype, + storage_type, + memory_layout, + DataGenType::ZEROS); + test_case.add_output_spec(output); + + if (dtype == vkapi::kHalf) { + test_case.set_abs_tolerance(1e-1f); + test_case.set_rel_tolerance(1e-2f); + } else { + test_case.set_abs_tolerance(1e-3f); + test_case.set_rel_tolerance(1e-3f); + } + + test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); + + return test_case; +} + +int64_t conv2d_flop_calculator(const TestCase& test_case) { + auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); + auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); + auto output_sizes = test_case.outputs()[0].get_tensor_sizes(); + + int64_t N = output_sizes[0]; + int64_t C_out = output_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; + int64_t C_in = input_sizes[1]; + int64_t K_h = weight_sizes[2]; + int64_t K_w = weight_sizes[3]; + + return 2 * N * C_out * C_in * H_out * W_out * K_h * K_w; +} + +int64_t conv2d_dw_flop_calculator(const TestCase& test_case) { + auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); + auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); + auto output_sizes = test_case.outputs()[0].get_tensor_sizes(); + + int64_t N = output_sizes[0]; + int64_t C = output_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; + int64_t K_h = weight_sizes[2]; + int64_t K_w = weight_sizes[3]; + + // Each output element: K_h * K_w multiplies + (K_h * K_w - 1) adds + return 2 * N * C * H_out * W_out * K_h * K_w; +} + +int64_t conv2d_pw_flop_calculator(const TestCase& test_case) { + auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); + auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); + + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t H = input_sizes[2]; + int64_t W = input_sizes[3]; + int64_t C_out = weight_sizes[0]; + + return 2 * N * C_out * C_in * H * W; +} + +// Reference implementation for general conv2d (groups=1). +// +// Supports both FP32 and (small-shape) FP16 inputs. The math is always done in +// float; for FP16 the master input/weight/bias values are dequantized from +// their half storage via get_element(), and the resulting float reference is +// compared against the dequantized GPU output by validate_against_reference(). +// +// FP16 accumulation error grows with K (= K_h * K_w * C_in). For large-K PERF +// shapes the FP32 reference would diverge from the GPU's FP16 accumulation +// enough to trip even the relaxed half tolerance, producing false failures, so +// those are intentionally left timing-only: this function throws +// std::invalid_argument, which execute_test_cases() catches to skip the +// correctness check (ref_computed stays false) while still benchmarking. +void conv2d_reference_impl(TestCase& test_case) { + const ValueSpec& input = test_case.inputs()[0]; + const ValueSpec& weight = test_case.inputs()[1]; + const ValueSpec& bias_spec = test_case.inputs()[2]; + ValueSpec& output = test_case.outputs()[0]; + + if (input.dtype != vkapi::kFloat && input.dtype != vkapi::kHalf) { + throw std::invalid_argument("Reference only supports float and half"); + } + + auto input_sizes = input.get_tensor_sizes(); + auto weight_sizes = weight.get_tensor_sizes(); + auto output_sizes = output.get_tensor_sizes(); + + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t H_in = input_sizes[2]; + int64_t W_in = input_sizes[3]; + int64_t C_out = weight_sizes[0]; + int64_t K_h = weight_sizes[2]; + int64_t K_w = weight_sizes[3]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; + + // For FP16, only compute a reference for small (ACCU) shapes where K is small + // enough that FP32-vs-FP16 accumulation error stays within the half + // tolerance. Large-K PERF half shapes stay timing-only via the throw below. + // The predicate mirrors create_conv2d_test_case's is_perf classification. + if (input.dtype == vkapi::kHalf) { + const bool is_perf = conv2d_is_perf_shape(C_in, C_out, H_in, W_in); + if (is_perf) { + throw std::invalid_argument( + "Half reference skipped for large-K PERF shape (timing-only)"); + } + } + + int64_t stride_h = test_case.inputs()[3].get_int_value(); + int64_t stride_w = test_case.inputs()[4].get_int_value(); + int64_t padding_h = test_case.inputs()[5].get_int_value(); + int64_t padding_w = test_case.inputs()[6].get_int_value(); + int64_t dilation_h = test_case.inputs()[7].get_int_value(); + int64_t dilation_w = test_case.inputs()[8].get_int_value(); + + // get_element() materializes a float regardless of dtype (it dequantizes + // half master data), so the same loop body serves both FP32 and FP16. + auto& ref_data = output.get_ref_float_data(); + ref_data.resize(N * C_out * H_out * W_out, 0.0f); + + for (int64_t n = 0; n < N; ++n) { + for (int64_t co = 0; co < C_out; ++co) { + for (int64_t oh = 0; oh < H_out; ++oh) { + for (int64_t ow = 0; ow < W_out; ++ow) { + float sum = 0.0f; + for (int64_t ci = 0; ci < C_in; ++ci) { + for (int64_t kh = 0; kh < K_h; ++kh) { + for (int64_t kw = 0; kw < K_w; ++kw) { + int64_t ih = oh * stride_h - padding_h + kh * dilation_h; + int64_t iw = ow * stride_w - padding_w + kw * dilation_w; + if (ih >= 0 && ih < H_in && iw >= 0 && iw < W_in) { + float in_val = input.get_element( + n * (C_in * H_in * W_in) + ci * (H_in * W_in) + + ih * W_in + iw); + // weight is [C_out, C_in, K_h, K_w] + float w_val = weight.get_element( + co * (C_in * K_h * K_w) + ci * (K_h * K_w) + kh * K_w + + kw); + sum += in_val * w_val; + } + } + } + } + if (!bias_spec.is_none()) { + sum += bias_spec.get_element(co); + } + ref_data + [n * (C_out * H_out * W_out) + co * (H_out * W_out) + oh * W_out + + ow] = sum; + } + } + } + } +} + +// Reference implementation for depthwise conv2d +void conv2d_dw_reference_impl(TestCase& test_case) { + const ValueSpec& input = test_case.inputs()[0]; + const ValueSpec& weight = test_case.inputs()[1]; + const ValueSpec& bias_spec = test_case.inputs()[2]; + ValueSpec& output = test_case.outputs()[0]; + + if (input.dtype != vkapi::kFloat) { + throw std::invalid_argument("Reference only supports float"); + } + + auto input_sizes = input.get_tensor_sizes(); + auto weight_sizes = weight.get_tensor_sizes(); + auto output_sizes = output.get_tensor_sizes(); + + int64_t N = input_sizes[0]; + int64_t C = input_sizes[1]; + int64_t H_in = input_sizes[2]; + int64_t W_in = input_sizes[3]; + int64_t K_h = weight_sizes[2]; + int64_t K_w = weight_sizes[3]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; + + int64_t stride_h = test_case.inputs()[3].get_int_value(); + int64_t stride_w = test_case.inputs()[4].get_int_value(); + int64_t padding_h = test_case.inputs()[5].get_int_value(); + int64_t padding_w = test_case.inputs()[6].get_int_value(); + int64_t dilation_h = test_case.inputs()[7].get_int_value(); + int64_t dilation_w = test_case.inputs()[8].get_int_value(); + + auto& input_data = input.get_float_data(); + auto& weight_data = weight.get_float_data(); + auto& ref_data = output.get_ref_float_data(); + ref_data.resize(N * C * H_out * W_out, 0.0f); + + for (int64_t n = 0; n < N; ++n) { + for (int64_t c = 0; c < C; ++c) { + for (int64_t oh = 0; oh < H_out; ++oh) { + for (int64_t ow = 0; ow < W_out; ++ow) { + float sum = 0.0f; + for (int64_t kh = 0; kh < K_h; ++kh) { + for (int64_t kw = 0; kw < K_w; ++kw) { + int64_t ih = oh * stride_h - padding_h + kh * dilation_h; + int64_t iw = ow * stride_w - padding_w + kw * dilation_w; + if (ih >= 0 && ih < H_in && iw >= 0 && iw < W_in) { + float in_val = input_data + [n * (C * H_in * W_in) + c * (H_in * W_in) + ih * W_in + + iw]; + // weight is [C, 1, K_h, K_w] + float w_val = weight_data[c * (K_h * K_w) + kh * K_w + kw]; + sum += in_val * w_val; + } + } + } + if (!bias_spec.is_none()) { + auto& bias_data = bias_spec.get_float_data(); + sum += bias_data[c]; + } + ref_data + [n * (C * H_out * W_out) + c * (H_out * W_out) + oh * W_out + + ow] = sum; + } + } + } + } +} + +// Reference implementation: pointwise conv2d is essentially a matmul +// output[n][c_out][h][w] = bias[c_out] + +// sum_over_c_in(input[n][c_in][h][w] * weight[c_out][c_in][0][0]) +void conv2d_pw_reference_impl(TestCase& test_case) { + // input[0], weight[1], bias[2], impl_selector[3] + const ValueSpec& input = test_case.inputs()[0]; + const ValueSpec& weight = test_case.inputs()[1]; + const ValueSpec& bias_spec = test_case.inputs()[2]; + ValueSpec& output = test_case.outputs()[0]; + + if (input.dtype != vkapi::kFloat) { + throw std::invalid_argument("Reference only supports float"); + } + + auto input_sizes = input.get_tensor_sizes(); + auto weight_sizes = weight.get_tensor_sizes(); + + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t H = input_sizes[2]; + int64_t W = input_sizes[3]; + int64_t C_out = weight_sizes[0]; + + auto& input_data = input.get_float_data(); + auto& weight_data = weight.get_float_data(); + auto& ref_data = output.get_ref_float_data(); + ref_data.resize(N * C_out * H * W, 0.0f); + + for (int64_t n = 0; n < N; ++n) { + for (int64_t co = 0; co < C_out; ++co) { + for (int64_t h = 0; h < H; ++h) { + for (int64_t w = 0; w < W; ++w) { + float sum = 0.0f; + for (int64_t ci = 0; ci < C_in; ++ci) { + float in_val = + input_data[n * (C_in * H * W) + ci * (H * W) + h * W + w]; + // weight is [C_out, C_in, 1, 1] + float w_val = weight_data[co * C_in + ci]; + sum += in_val * w_val; + } + if (!bias_spec.is_none()) { + auto& bias_data = bias_spec.get_float_data(); + sum += bias_data[co]; + } + ref_data[n * (C_out * H * W) + co * (H * W) + h * W + w] = sum; + } + } + } + } +} + +} // namespace conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.h b/backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.h new file mode 100644 index 00000000000..858e8b22fd7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.h @@ -0,0 +1,101 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv2d { + +constexpr int64_t kRefDimSizeLimit = 64; + +// Conv2d shaders are texture-only and require channels-packed layout +constexpr utils::StorageType kStorageType = utils::kTexture3D; +constexpr utils::GPUMemoryLayout kMemoryLayout = utils::kChannelsPacked; + +struct InputDims { + int64_t N; + int64_t C; + int64_t H; + int64_t W; + + InputDims(int64_t n, int64_t c, int64_t h, int64_t w) + : N(n), C(c), H(h), W(w) {} +}; + +struct Conv2dTestConfig { + InputDims dims; + int64_t C_out; + KernelSize kernel; + Stride stride; + Padding padding; + Dilation dilation; + bool has_bias; +}; + +struct Conv2dDwConfig { + InputDims dims; + KernelSize kernel; + Stride stride; + Padding padding; + Dilation dilation; + bool has_bias; +}; + +struct Conv2dPwConfig { + int64_t N; + int64_t C_in; + int64_t C_out; + int64_t H; + int64_t W; + bool has_bias; +}; + +bool conv2d_is_perf_shape(int64_t C_in, int64_t C_out, int64_t H, int64_t W); + +TestCase create_conv2d_test_case( + const Conv2dTestConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout, + const std::string& impl_selector = ""); + +TestCase create_conv2d_dw_test_case( + const Conv2dDwConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout, + const std::string& impl_selector = ""); + +TestCase create_conv2d_pw_test_case( + const Conv2dPwConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout); + +void conv2d_reference_impl(TestCase& test_case); + +void conv2d_dw_reference_impl(TestCase& test_case); + +void conv2d_pw_reference_impl(TestCase& test_case); + +int64_t conv2d_flop_calculator(const TestCase& test_case); + +int64_t conv2d_dw_flop_calculator(const TestCase& test_case); + +int64_t conv2d_pw_flop_calculator(const TestCase& test_case); + +} // namespace conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d.cpp deleted file mode 100644 index 83a604345b8..00000000000 --- a/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d.cpp +++ /dev/null @@ -1,639 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include - -#include -#include - -#include -#include - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 64; - -struct InputDims { - int64_t N; - int64_t C; - int64_t H; - int64_t W; - - InputDims(int64_t n, int64_t c, int64_t h, int64_t w) - : N(n), C(c), H(h), W(w) {} -}; - -struct Conv2dTestConfig { - InputDims dims; - int64_t C_out; - KernelSize kernel; - Stride stride; - Padding padding; - Dilation dilation; - bool has_bias; -}; - -static int64_t calc_out_size( - int64_t in_size, - int64_t kernel_size, - int64_t stride, - int64_t padding, - int64_t dilation) { - return (in_size + 2 * padding - dilation * (kernel_size - 1) - 1) / stride + - 1; -} - -// Shared perf/skip classification used by both create_conv2d_test_case (to tag -// PERF vs ACCU) and conv2d_reference_impl (to gate the large-K FP16 reference -// check). A shape is "perf" if any dimension reaches kRefDimSizeLimit; the -// boundary is inclusive (>=) so a 64-wide dim counts as perf — FP16 -// accumulation error at K = K_h * K_w * C_in for such shapes can exceed the -// half tolerance and false-fail. Keep both call sites on this single helper to -// avoid the two predicates drifting apart. -static bool -conv2d_is_perf_shape(int64_t C_in, int64_t C_out, int64_t H, int64_t W) { - return C_in >= kRefDimSizeLimit || C_out >= kRefDimSizeLimit || - H >= kRefDimSizeLimit || W >= kRefDimSizeLimit; -} - -static TestCase create_conv2d_test_case( - const Conv2dTestConfig& config, - vkapi::ScalarType dtype, - utils::StorageType storage_type, - utils::GPUMemoryLayout memory_layout, - const std::string& impl_selector = "") { - TestCase test_case; - - bool is_perf = conv2d_is_perf_shape( - config.dims.C, config.C_out, config.dims.H, config.dims.W); - - std::string prefix = is_perf ? "PERF" : "ACCU"; - std::string storage_str = repr_str(storage_type, memory_layout); - std::string dtype_str = dtype_short(dtype); - std::string bias_str = config.has_bias ? "+bias" : ""; - - int64_t H_out = calc_out_size( - config.dims.H, - config.kernel.h, - config.stride.h, - config.padding.h, - config.dilation.h); - int64_t W_out = calc_out_size( - config.dims.W, - config.kernel.w, - config.stride.w, - config.padding.w, - config.dilation.w); - - std::string shape = "[" + std::to_string(config.dims.N) + "," + - std::to_string(config.dims.C) + "," + std::to_string(config.dims.H) + - "," + std::to_string(config.dims.W) + "]->[" + - std::to_string(config.C_out) + "] k" + std::to_string(config.kernel.h) + - "x" + std::to_string(config.kernel.w) + " s" + - std::to_string(config.stride.h) + " p" + - std::to_string(config.padding.h) + " d" + - std::to_string(config.dilation.h); - - std::string suffix = bias_str; - if (!impl_selector.empty()) { - if (!suffix.empty()) { - suffix += " "; - } - suffix += "[" + impl_selector + "]"; - } - - std::string name = - make_test_label(prefix, dtype_str, dtype_str, shape, storage_str, suffix); - - test_case.set_name(name); - test_case.set_operator_name("test_etvk.test_conv2d.default"); - - // Input tensor [N, C_in, H, W] - ValueSpec input( - {config.dims.N, config.dims.C, config.dims.H, config.dims.W}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - - // Weight tensor [C_out, C_in, K_h, K_w] - constant - ValueSpec weight( - {config.C_out, config.dims.C, config.kernel.h, config.kernel.w}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - weight.set_constant(true); - - test_case.add_input_spec(input); - test_case.add_input_spec(weight); - - // Bias (or none) - if (config.has_bias) { - ValueSpec bias( - {config.C_out}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - bias.set_constant(true); - test_case.add_input_spec(bias); - } else { - ValueSpec none_bias(static_cast(0)); - none_bias.set_none(true); - test_case.add_input_spec(none_bias); - } - - // stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w - test_case.add_input_spec(ValueSpec(static_cast(config.stride.h))); - test_case.add_input_spec(ValueSpec(static_cast(config.stride.w))); - test_case.add_input_spec(ValueSpec(static_cast(config.padding.h))); - test_case.add_input_spec(ValueSpec(static_cast(config.padding.w))); - test_case.add_input_spec(ValueSpec(static_cast(config.dilation.h))); - test_case.add_input_spec(ValueSpec(static_cast(config.dilation.w))); - - // impl_selector string - test_case.add_input_spec(ValueSpec::make_string(impl_selector)); - - // Output tensor [N, C_out, H_out, W_out] - ValueSpec output( - {config.dims.N, config.C_out, H_out, W_out}, - dtype, - storage_type, - memory_layout, - DataGenType::ZEROS); - test_case.add_output_spec(output); - - if (dtype == vkapi::kHalf) { - test_case.set_abs_tolerance(1e-1f); - test_case.set_rel_tolerance(1e-2f); - } else { - test_case.set_abs_tolerance(1e-3f); - test_case.set_rel_tolerance(1e-3f); - } - - test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); - - return test_case; -} - -// Reference implementation for general conv2d (groups=1). -// -// Supports both FP32 and (small-shape) FP16 inputs. The math is always done in -// float; for FP16 the master input/weight/bias values are dequantized from -// their half storage via get_element(), and the resulting float reference is -// compared against the dequantized GPU output by validate_against_reference(). -// -// FP16 accumulation error grows with K (= K_h * K_w * C_in). For large-K PERF -// shapes the FP32 reference would diverge from the GPU's FP16 accumulation -// enough to trip even the relaxed half tolerance, producing false failures, so -// those are intentionally left timing-only: this function throws -// std::invalid_argument, which execute_test_cases() catches to skip the -// correctness check (ref_computed stays false) while still benchmarking. -static void conv2d_reference_impl(TestCase& test_case) { - const ValueSpec& input = test_case.inputs()[0]; - const ValueSpec& weight = test_case.inputs()[1]; - const ValueSpec& bias_spec = test_case.inputs()[2]; - ValueSpec& output = test_case.outputs()[0]; - - if (input.dtype != vkapi::kFloat && input.dtype != vkapi::kHalf) { - throw std::invalid_argument("Reference only supports float and half"); - } - - auto input_sizes = input.get_tensor_sizes(); - auto weight_sizes = weight.get_tensor_sizes(); - auto output_sizes = output.get_tensor_sizes(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t H_in = input_sizes[2]; - int64_t W_in = input_sizes[3]; - int64_t C_out = weight_sizes[0]; - int64_t K_h = weight_sizes[2]; - int64_t K_w = weight_sizes[3]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - // For FP16, only compute a reference for small (ACCU) shapes where K is small - // enough that FP32-vs-FP16 accumulation error stays within the half - // tolerance. Large-K PERF half shapes stay timing-only via the throw below. - // The predicate mirrors create_conv2d_test_case's is_perf classification. - if (input.dtype == vkapi::kHalf) { - const bool is_perf = conv2d_is_perf_shape(C_in, C_out, H_in, W_in); - if (is_perf) { - throw std::invalid_argument( - "Half reference skipped for large-K PERF shape (timing-only)"); - } - } - - int64_t stride_h = test_case.inputs()[3].get_int_value(); - int64_t stride_w = test_case.inputs()[4].get_int_value(); - int64_t padding_h = test_case.inputs()[5].get_int_value(); - int64_t padding_w = test_case.inputs()[6].get_int_value(); - int64_t dilation_h = test_case.inputs()[7].get_int_value(); - int64_t dilation_w = test_case.inputs()[8].get_int_value(); - - // get_element() materializes a float regardless of dtype (it dequantizes - // half master data), so the same loop body serves both FP32 and FP16. - auto& ref_data = output.get_ref_float_data(); - ref_data.resize(N * C_out * H_out * W_out, 0.0f); - - for (int64_t n = 0; n < N; ++n) { - for (int64_t co = 0; co < C_out; ++co) { - for (int64_t oh = 0; oh < H_out; ++oh) { - for (int64_t ow = 0; ow < W_out; ++ow) { - float sum = 0.0f; - for (int64_t ci = 0; ci < C_in; ++ci) { - for (int64_t kh = 0; kh < K_h; ++kh) { - for (int64_t kw = 0; kw < K_w; ++kw) { - int64_t ih = oh * stride_h - padding_h + kh * dilation_h; - int64_t iw = ow * stride_w - padding_w + kw * dilation_w; - if (ih >= 0 && ih < H_in && iw >= 0 && iw < W_in) { - float in_val = input.get_element( - n * (C_in * H_in * W_in) + ci * (H_in * W_in) + - ih * W_in + iw); - // weight is [C_out, C_in, K_h, K_w] - float w_val = weight.get_element( - co * (C_in * K_h * K_w) + ci * (K_h * K_w) + kh * K_w + - kw); - sum += in_val * w_val; - } - } - } - } - if (!bias_spec.is_none()) { - sum += bias_spec.get_element(co); - } - ref_data - [n * (C_out * H_out * W_out) + co * (H_out * W_out) + oh * W_out + - ow] = sum; - } - } - } - } -} - -static std::vector generate_conv2d_test_cases() { - std::vector test_cases; - - std::vector storage_types = {utils::kTexture3D}; - utils::GPUMemoryLayout layout = utils::kChannelsPacked; - - // Accuracy shapes (small enough for float reference validation) - std::vector accuracy_configs = { - // 3x3 stride=1 pad=1 same-channels (the bottleneck pattern in TinyCNN) - {InputDims(1, 8, 8, 8), - 8, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 8, 8, 8), - 8, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - {InputDims(1, 16, 16, 16), - 16, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // 3x3 stride=2 (downsample) with channel expansion - {InputDims(1, 8, 16, 16), - 16, - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - // 3x3 stride=1 with channel reduction - {InputDims(1, 16, 8, 8), - 8, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - // Non-multiple-of-4 channels - {InputDims(1, 11, 8, 8), - 13, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - // 3-channel input (like RGB stem) - {InputDims(1, 3, 16, 16), - 8, - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - }; - - // TinyCNN depth estimator hotspots (from profiling - // UNTRAINED_TinyCNNDepthEstimatorRealTime_Vulkan.pte). - // Each entry lists (C_in, H, W) -> C_out, all 3x3 stride=1 pad=1 unless - // noted. Together the first 6 entries account for ~89% of all conv time. - std::vector perf_configs = { - // #1: 21.25% — (1,128,36,48)->(1,128,36,48) - {InputDims(1, 128, 36, 48), - 128, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // #2: 20.68% — (1,256,18,24)->(1,256,18,24) - {InputDims(1, 256, 18, 24), - 256, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // #3: 20.01% — (1,64,72,96)->(1,64,72,96) - {InputDims(1, 64, 72, 96), - 64, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // #4: 13.25% — (1,32,144,192)->(1,32,144,192) - {InputDims(1, 32, 144, 192), - 32, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // #5: 6.74% — (1,64,36,48)->(1,64,36,48) - {InputDims(1, 64, 36, 48), - 64, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // #6: 5.90% — (1,32,72,96)->(1,32,72,96) - {InputDims(1, 32, 72, 96), - 32, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // Secondary cases - // 3x3 stride=2 downsample with channel expansion: 1.52% - {InputDims(1, 32, 72, 96), - 128, - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - // 3x3 stride=1 same-shape, smaller spatial: 1.51% - {InputDims(1, 128, 18, 24), - 128, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // 3x3 stride=1, channel reduction - {InputDims(1, 128, 18, 24), - 64, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 64, 36, 48), - 32, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - // 3x3 stride=2 downsample, same channels - {InputDims(1, 32, 72, 96), - 32, - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 64, 36, 48), - 64, - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - // RGB stem - {InputDims(1, 3, 144, 192), - 32, - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - }; - - // Small shapes used to exercise each im2col intermediate-storage variant - // (buffer / texture2d / texture3d) deterministically and independently of - // the device's auto-selection. All dims < kRefDimSizeLimit so both the FP32 - // and FP16 references validate them. The first two shapes keep M (= H_out * - // W_out) tiny, so the im2col intermediate [1, K_total, H_out, W_out] is small - // and the forced texture3d variant never exercises the large-M texture3d - // Z-layout the auto-selector actually falls back to. The third shape (48x48) - // is included specifically so the forced im2col_tex3d variant exercises the - // texture3d layout (K4 along Z, many spatial rows) at a non-trivial M, while - // still keeping every dim under kRefDimSizeLimit for reference validation. - std::vector per_variant_configs = { - // 3x3 s1 p1, channels multiple of 4 - {InputDims(1, 16, 16, 16), - 16, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // Non-multiple-of-4 channels exercise the Cin padding path - {InputDims(1, 11, 12, 12), - 13, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - // Larger spatial extent (M = 48*48 = 2304) exercises the texture3d im2col - // layout [1, K_total, H_out, W_out] with K4 along Z at a non-trivial M, - // while all dims stay < kRefDimSizeLimit so both FP32 and FP16 references - // validate it. C_in=16 keeps K = 3*3*16 = 144 (same as the 16x16 case) - // so FP16 accumulation stays within tolerance. - {InputDims(1, 16, 48, 48), - 16, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - }; - - // Boundary pair straddling the should_use_conv2d_im2col() c_out >= 128 - // routing threshold. Spatial dims are tiny (8x8) so the FP32 float reference - // stays cheap, but c_out = 64 / 128 are both >= kRefDimSizeLimit, so these - // get the PERF label. FP32 PERF cases are still numerically VERIFIED (the - // reference's invalid_argument throw that skips the check only fires for - // half), so both implementations are cross-checked against the float - // reference at the boundary. Run all three impls: at c_out = 64 the heuristic - // ("") picks direct on Adreno / im2col on Mali; at c_out = 128 it picks - // im2col on both — and "direct"/"im2col" force each path regardless, proving - // the two implementations agree at the boundary on either device. - std::vector boundary_configs = { - // c_out = 64 (< 128): below the threshold - {InputDims(1, 16, 8, 8), - 64, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - // c_out = 128 (== 128): at/above the threshold - {InputDims(1, 16, 8, 8), - 128, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - }; - - // Implementation variants exercised for every small ACCU shape: - // "" -> heuristic-routed (should_use_conv2d_im2col picks direct on - // Adreno for small c_out, im2col on Mali) - // "im2col" -> forced im2col/GEMM path - // "direct" -> forced direct sliding-window path (conv2d_direct_impl) - // Including "direct" guarantees the direct shader gets reference-checked on - // BOTH devices — without it, Mali would always route "" to im2col and never - // exercise the direct path. - const std::vector impls = {"", "im2col", "direct"}; - // Forced-storage im2col variants for the per-variant ACCU coverage. - const std::vector forced_storage_impls = { - "im2col_buffer", "im2col_tex2d", "im2col_tex3d"}; - - // Generate accuracy test cases for all impls and both dtypes. FP16 small - // shapes get a real reference check (gated in conv2d_reference_impl); we run - // both dtypes so we catch correctness regressions in either path. Large-K - // half stays timing-only via the reference's PERF-shape throw. - const std::vector accu_dtypes = { - vkapi::kFloat, vkapi::kHalf}; - for (const auto& config : accuracy_configs) { - for (auto st : storage_types) { - for (auto dtype : accu_dtypes) { - for (const auto& impl : impls) { - test_cases.push_back( - create_conv2d_test_case(config, dtype, st, layout, impl)); - } - } - } - } - - // Generate per-variant forced-storage ACCU cases (FP32 and FP16) so all - // three im2col intermediate-storage variants get deterministic, - // device-independent, reference-checked coverage at small K. - for (const auto& config : per_variant_configs) { - for (auto st : storage_types) { - for (auto dtype : accu_dtypes) { - for (const auto& impl : forced_storage_impls) { - test_cases.push_back( - create_conv2d_test_case(config, dtype, st, layout, impl)); - } - } - } - } - - // Generate the c_out boundary pair (FP32 only) through all three impls. - // FP32 PERF cases are reference-VERIFIED, so the direct and im2col paths are - // both cross-checked against the float reference at the routing threshold. - for (const auto& config : boundary_configs) { - for (auto st : storage_types) { - for (const auto& impl : impls) { - test_cases.push_back( - create_conv2d_test_case(config, vkapi::kFloat, st, layout, impl)); - } - } - } - - // Generate performance test cases (float and half) for all impls. - for (const auto& config : perf_configs) { - std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; - for (auto dtype : dtypes) { - for (auto st : storage_types) { - for (const auto& impl : impls) { - test_cases.push_back( - create_conv2d_test_case(config, dtype, st, layout, impl)); - } - } - } - } - - return test_cases; -} - -static int64_t conv2d_flop_calculator(const TestCase& test_case) { - auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); - auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); - auto output_sizes = test_case.outputs()[0].get_tensor_sizes(); - - int64_t N = output_sizes[0]; - int64_t C_out = output_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - int64_t C_in = input_sizes[1]; - int64_t K_h = weight_sizes[2]; - int64_t K_w = weight_sizes[3]; - - return 2 * N * C_out * C_in * H_out * W_out * K_h * K_w; -} - -static void reference_impl(TestCase& test_case) { - conv2d_reference_impl(test_case); -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "General Conv2d (SlidingWindow, groups=1) Benchmark" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - execute_test_cases( - generate_conv2d_test_cases, - conv2d_flop_calculator, - "Conv2d", - /*warmup_runs = */ 5, - /*benchmark_runs = */ 20, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_dw.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_dw.cpp deleted file mode 100644 index 4ba7a3d6286..00000000000 --- a/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_dw.cpp +++ /dev/null @@ -1,439 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include - -#include -#include - -#include -#include - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 64; - -struct InputDims { - int64_t N; - int64_t C; - int64_t H; - int64_t W; - - InputDims(int64_t n, int64_t c, int64_t h, int64_t w) - : N(n), C(c), H(h), W(w) {} -}; - -struct Conv2dDwConfig { - InputDims dims; - KernelSize kernel; - Stride stride; - Padding padding; - Dilation dilation; - bool has_bias; -}; - -static int64_t calc_out_size( - int64_t in_size, - int64_t kernel_size, - int64_t stride, - int64_t padding, - int64_t dilation) { - return (in_size + 2 * padding - dilation * (kernel_size - 1) - 1) / stride + - 1; -} - -static TestCase create_conv2d_dw_test_case( - const Conv2dDwConfig& config, - vkapi::ScalarType dtype, - utils::StorageType storage_type, - utils::GPUMemoryLayout memory_layout, - const std::string& impl_selector = "") { - TestCase test_case; - - bool is_perf = config.dims.C > kRefDimSizeLimit || - config.dims.H > kRefDimSizeLimit || config.dims.W > kRefDimSizeLimit; - - std::string prefix = is_perf ? "PERF" : "ACCU"; - std::string storage_str = repr_str(storage_type, memory_layout); - std::string dtype_str = dtype_short(dtype); - std::string bias_str = config.has_bias ? "+bias" : ""; - - int64_t H_out = calc_out_size( - config.dims.H, - config.kernel.h, - config.stride.h, - config.padding.h, - config.dilation.h); - int64_t W_out = calc_out_size( - config.dims.W, - config.kernel.w, - config.stride.w, - config.padding.w, - config.dilation.w); - (void)H_out; - (void)W_out; - - // groups for depthwise conv2d == number of input channels - std::string shape = "[" + std::to_string(config.dims.N) + "," + - std::to_string(config.dims.C) + "," + std::to_string(config.dims.H) + - "," + std::to_string(config.dims.W) + "] k" + - std::to_string(config.kernel.h) + " s" + std::to_string(config.stride.h) + - " p" + std::to_string(config.padding.h) + " d" + - std::to_string(config.dilation.h) + " g" + std::to_string(config.dims.C); - - std::string suffix = bias_str; - if (!impl_selector.empty()) { - if (!suffix.empty()) { - suffix += " "; - } - suffix += "[" + impl_selector + "]"; - } - - std::string name = - make_test_label(prefix, dtype_str, dtype_str, shape, storage_str, suffix); - - test_case.set_name(name); - test_case.set_operator_name("test_etvk.test_conv2d_dw.default"); - - // Input tensor [N, C, H, W] - ValueSpec input( - {config.dims.N, config.dims.C, config.dims.H, config.dims.W}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - - // Weight tensor [C, 1, K_h, K_w] - constant - ValueSpec weight( - {config.dims.C, 1, config.kernel.h, config.kernel.w}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - weight.set_constant(true); - - test_case.add_input_spec(input); - test_case.add_input_spec(weight); - - // Bias (or none) - if (config.has_bias) { - ValueSpec bias( - {config.dims.C}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - bias.set_constant(true); - test_case.add_input_spec(bias); - } else { - ValueSpec none_bias(static_cast(0)); - none_bias.set_none(true); - test_case.add_input_spec(none_bias); - } - - // stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w - test_case.add_input_spec(ValueSpec(static_cast(config.stride.h))); - test_case.add_input_spec(ValueSpec(static_cast(config.stride.w))); - test_case.add_input_spec(ValueSpec(static_cast(config.padding.h))); - test_case.add_input_spec(ValueSpec(static_cast(config.padding.w))); - test_case.add_input_spec(ValueSpec(static_cast(config.dilation.h))); - test_case.add_input_spec(ValueSpec(static_cast(config.dilation.w))); - - // impl_selector string - test_case.add_input_spec(ValueSpec::make_string(impl_selector)); - - // Output tensor [N, C, H_out, W_out] - ValueSpec output( - {config.dims.N, config.dims.C, H_out, W_out}, - dtype, - storage_type, - memory_layout, - DataGenType::ZEROS); - test_case.add_output_spec(output); - - if (dtype == vkapi::kHalf) { - test_case.set_abs_tolerance(1e-1f); - test_case.set_rel_tolerance(1e-2f); - } else { - test_case.set_abs_tolerance(1e-3f); - test_case.set_rel_tolerance(1e-3f); - } - - test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); - - return test_case; -} - -// Reference implementation for depthwise conv2d -static void conv2d_dw_reference_impl(TestCase& test_case) { - const ValueSpec& input = test_case.inputs()[0]; - const ValueSpec& weight = test_case.inputs()[1]; - const ValueSpec& bias_spec = test_case.inputs()[2]; - ValueSpec& output = test_case.outputs()[0]; - - if (input.dtype != vkapi::kFloat) { - throw std::invalid_argument("Reference only supports float"); - } - - auto input_sizes = input.get_tensor_sizes(); - auto weight_sizes = weight.get_tensor_sizes(); - auto output_sizes = output.get_tensor_sizes(); - - int64_t N = input_sizes[0]; - int64_t C = input_sizes[1]; - int64_t H_in = input_sizes[2]; - int64_t W_in = input_sizes[3]; - int64_t K_h = weight_sizes[2]; - int64_t K_w = weight_sizes[3]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - int64_t stride_h = test_case.inputs()[3].get_int_value(); - int64_t stride_w = test_case.inputs()[4].get_int_value(); - int64_t padding_h = test_case.inputs()[5].get_int_value(); - int64_t padding_w = test_case.inputs()[6].get_int_value(); - int64_t dilation_h = test_case.inputs()[7].get_int_value(); - int64_t dilation_w = test_case.inputs()[8].get_int_value(); - - auto& input_data = input.get_float_data(); - auto& weight_data = weight.get_float_data(); - auto& ref_data = output.get_ref_float_data(); - ref_data.resize(N * C * H_out * W_out, 0.0f); - - for (int64_t n = 0; n < N; ++n) { - for (int64_t c = 0; c < C; ++c) { - for (int64_t oh = 0; oh < H_out; ++oh) { - for (int64_t ow = 0; ow < W_out; ++ow) { - float sum = 0.0f; - for (int64_t kh = 0; kh < K_h; ++kh) { - for (int64_t kw = 0; kw < K_w; ++kw) { - int64_t ih = oh * stride_h - padding_h + kh * dilation_h; - int64_t iw = ow * stride_w - padding_w + kw * dilation_w; - if (ih >= 0 && ih < H_in && iw >= 0 && iw < W_in) { - float in_val = input_data - [n * (C * H_in * W_in) + c * (H_in * W_in) + ih * W_in + - iw]; - // weight is [C, 1, K_h, K_w] - float w_val = weight_data[c * (K_h * K_w) + kh * K_w + kw]; - sum += in_val * w_val; - } - } - } - if (!bias_spec.is_none()) { - auto& bias_data = bias_spec.get_float_data(); - sum += bias_data[c]; - } - ref_data - [n * (C * H_out * W_out) + c * (H_out * W_out) + oh * W_out + - ow] = sum; - } - } - } - } -} - -static std::vector generate_conv2d_dw_test_cases() { - std::vector test_cases; - - std::vector storage_types = {utils::kTexture3D}; - utils::GPUMemoryLayout layout = utils::kChannelsPacked; - - // Accuracy shapes (small enough for float reference validation) - std::vector accuracy_configs = { - {InputDims(1, 8, 16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 8, 16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - {InputDims(1, 8, 16, 16), - KernelSize(5, 5), - Stride(1, 1), - Padding(2, 2), - Dilation(1, 1), - false}, - {InputDims(1, 8, 16, 16), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - // Non-multiple-of-4 channels - {InputDims(1, 11, 16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 3, 16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - }; - - // EdgeTAM depthwise shapes (from profiling data) - std::vector perf_configs = { - // Backbone stem and early stages - {InputDims(1, 24, 512, 512), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 48, 256, 256), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 48, 256, 256), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 96, 128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 96, 128, 128), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 192, 64, 64), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 192, 64, 64), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 384, 32, 32), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - // 5x5 kernels - {InputDims(1, 48, 256, 256), - KernelSize(5, 5), - Stride(1, 1), - Padding(2, 2), - Dilation(1, 1), - false}, - {InputDims(1, 96, 128, 128), - KernelSize(5, 5), - Stride(1, 1), - Padding(2, 2), - Dilation(1, 1), - false}, - // FPN/Neck - {InputDims(1, 256, 256, 256), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 256, 128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - }; - - // Generate accuracy test cases (float only) - for (const auto& config : accuracy_configs) { - for (auto st : storage_types) { - test_cases.push_back( - create_conv2d_dw_test_case(config, vkapi::kFloat, st, layout)); - } - } - - // Generate performance test cases (float and half) - for (const auto& config : perf_configs) { - std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; - for (auto dtype : dtypes) { - for (auto st : storage_types) { - // Auto-selection (empty impl_selector) - test_cases.push_back( - create_conv2d_dw_test_case(config, dtype, st, layout)); - - // Force b4x2 variant - test_cases.push_back( - create_conv2d_dw_test_case(config, dtype, st, layout, "b4x2")); - - // Force b1x1 variant (only for 3x3 kernels; for 5x5 it falls back - // to default, but we still generate it to test the fallback path) - test_cases.push_back( - create_conv2d_dw_test_case(config, dtype, st, layout, "b1x1")); - } - } - } - - return test_cases; -} - -static int64_t conv2d_dw_flop_calculator(const TestCase& test_case) { - auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); - auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); - auto output_sizes = test_case.outputs()[0].get_tensor_sizes(); - - int64_t N = output_sizes[0]; - int64_t C = output_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - int64_t K_h = weight_sizes[2]; - int64_t K_w = weight_sizes[3]; - - // Each output element: K_h * K_w multiplies + (K_h * K_w - 1) adds - return 2 * N * C * H_out * W_out * K_h * K_w; -} - -static void reference_impl(TestCase& test_case) { - conv2d_dw_reference_impl(test_case); -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Depthwise Conv2d Benchmark" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_conv2d_dw_test_cases, - conv2d_dw_flop_calculator, - "Conv2dDW", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_pw.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_pw.cpp deleted file mode 100644 index da38a744417..00000000000 --- a/backends/vulkan/test/ops/benchmark/cases/conv2d/test_conv2d_pw.cpp +++ /dev/null @@ -1,267 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include - -#include -#include - -#include - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 64; - -struct Conv2dPwConfig { - int64_t N; - int64_t C_in; - int64_t C_out; - int64_t H; - int64_t W; - bool has_bias; -}; - -static TestCase create_conv2d_pw_test_case( - const Conv2dPwConfig& config, - vkapi::ScalarType dtype, - utils::StorageType storage_type, - utils::GPUMemoryLayout memory_layout) { - TestCase test_case; - - bool is_perf = config.C_in > kRefDimSizeLimit || - config.C_out > kRefDimSizeLimit || config.H > kRefDimSizeLimit || - config.W > kRefDimSizeLimit; - - std::string prefix = is_perf ? "PERF" : "ACCU"; - std::string storage_str = repr_str(storage_type, memory_layout); - std::string dtype_str = dtype_short(dtype); - std::string bias_str = config.has_bias ? "+bias" : ""; - - // Pointwise conv2d: kernel 1x1, stride 1, pad 0, dilation 1, groups 1 - std::string shape = "[" + std::to_string(config.N) + "," + - std::to_string(config.C_in) + "," + std::to_string(config.H) + "," + - std::to_string(config.W) + "]x[" + std::to_string(config.C_out) + "," + - std::to_string(config.C_in) + ",1,1] s1 p0 d1 g1"; - - std::string name = make_test_label( - prefix, dtype_str, dtype_str, shape, storage_str, bias_str); - - test_case.set_name(name); - test_case.set_operator_name("test_etvk.test_conv2d_pw.default"); - - // Input tensor [N, C_in, H, W] - ValueSpec input( - {config.N, config.C_in, config.H, config.W}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - - // Weight tensor [C_out, C_in, 1, 1] - constant - ValueSpec weight( - {config.C_out, config.C_in, 1, 1}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - weight.set_constant(true); - - test_case.add_input_spec(input); - test_case.add_input_spec(weight); - - // Bias (or none) - if (config.has_bias) { - ValueSpec bias( - {config.C_out}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - bias.set_constant(true); - test_case.add_input_spec(bias); - } else { - ValueSpec none_bias(static_cast(0)); - none_bias.set_none(true); - test_case.add_input_spec(none_bias); - } - - // impl_selector - ValueSpec impl_selector_spec = ValueSpec::make_string("default"); - test_case.add_input_spec(impl_selector_spec); - - // Output tensor [N, C_out, H, W] - ValueSpec output( - {config.N, config.C_out, config.H, config.W}, - dtype, - storage_type, - memory_layout, - DataGenType::ZEROS); - test_case.add_output_spec(output); - - if (dtype == vkapi::kHalf) { - test_case.set_abs_tolerance(1e-1f); - test_case.set_rel_tolerance(1e-2f); - } else { - test_case.set_abs_tolerance(1e-3f); - test_case.set_rel_tolerance(1e-3f); - } - - test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); - - return test_case; -} - -// Reference implementation: pointwise conv2d is essentially a matmul -// output[n][c_out][h][w] = bias[c_out] + -// sum_over_c_in(input[n][c_in][h][w] * weight[c_out][c_in][0][0]) -static void conv2d_pw_reference_impl(TestCase& test_case) { - // input[0], weight[1], bias[2], impl_selector[3] - const ValueSpec& input = test_case.inputs()[0]; - const ValueSpec& weight = test_case.inputs()[1]; - const ValueSpec& bias_spec = test_case.inputs()[2]; - ValueSpec& output = test_case.outputs()[0]; - - if (input.dtype != vkapi::kFloat) { - throw std::invalid_argument("Reference only supports float"); - } - - auto input_sizes = input.get_tensor_sizes(); - auto weight_sizes = weight.get_tensor_sizes(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t H = input_sizes[2]; - int64_t W = input_sizes[3]; - int64_t C_out = weight_sizes[0]; - - auto& input_data = input.get_float_data(); - auto& weight_data = weight.get_float_data(); - auto& ref_data = output.get_ref_float_data(); - ref_data.resize(N * C_out * H * W, 0.0f); - - for (int64_t n = 0; n < N; ++n) { - for (int64_t co = 0; co < C_out; ++co) { - for (int64_t h = 0; h < H; ++h) { - for (int64_t w = 0; w < W; ++w) { - float sum = 0.0f; - for (int64_t ci = 0; ci < C_in; ++ci) { - float in_val = - input_data[n * (C_in * H * W) + ci * (H * W) + h * W + w]; - // weight is [C_out, C_in, 1, 1] - float w_val = weight_data[co * C_in + ci]; - sum += in_val * w_val; - } - if (!bias_spec.is_none()) { - auto& bias_data = bias_spec.get_float_data(); - sum += bias_data[co]; - } - ref_data[n * (C_out * H * W) + co * (H * W) + h * W + w] = sum; - } - } - } - } -} - -static std::vector generate_conv2d_pw_test_cases() { - std::vector test_cases; - - // Conv2d shaders are texture-only and require channels-packed layout - std::vector storage_types = {utils::kTexture3D}; - utils::GPUMemoryLayout layout = utils::kChannelsPacked; - - // Accuracy shapes (small enough for float reference validation) - std::vector accuracy_configs = { - {1, 16, 32, 8, 8, false}, - {1, 32, 16, 8, 8, false}, - {1, 16, 32, 8, 8, true}, - {1, 48, 96, 16, 16, false}, - {1, 96, 48, 16, 16, false}, - // Non-multiple-of-4 channels - {1, 13, 27, 8, 8, false}, - {1, 33, 17, 8, 8, false}, - }; - - // EdgeTAM performance shapes - std::vector perf_configs = { - // EdgeTAM backbone stages - {1, 48, 96, 256, 256, false}, - {1, 96, 48, 256, 256, false}, - {1, 96, 192, 128, 128, false}, - {1, 192, 96, 128, 128, false}, - {1, 192, 384, 64, 64, false}, - {1, 384, 192, 64, 64, false}, - {1, 384, 768, 32, 32, false}, - {1, 768, 384, 32, 32, false}, - // EdgeTAM FPN/Neck - {1, 48, 256, 256, 256, false}, - {1, 256, 32, 256, 256, false}, - {1, 96, 256, 128, 128, false}, - {1, 256, 64, 128, 128, false}, - }; - - // Generate accuracy test cases (float only) - for (const auto& config : accuracy_configs) { - for (auto st : storage_types) { - test_cases.push_back( - create_conv2d_pw_test_case(config, vkapi::kFloat, st, layout)); - } - } - - // Generate performance test cases (float and half) - for (const auto& config : perf_configs) { - std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; - for (auto dtype : dtypes) { - for (auto st : storage_types) { - test_cases.push_back( - create_conv2d_pw_test_case(config, dtype, st, layout)); - } - } - } - - return test_cases; -} - -static int64_t conv2d_pw_flop_calculator(const TestCase& test_case) { - auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); - auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t H = input_sizes[2]; - int64_t W = input_sizes[3]; - int64_t C_out = weight_sizes[0]; - - return 2 * N * C_out * C_in * H * W; -} - -static void reference_impl(TestCase& test_case) { - conv2d_pw_reference_impl(test_case); -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Pointwise Conv2d (1x1) Benchmark" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_conv2d_pw_test_cases, - conv2d_pw_flop_calculator, - "Conv2dPW", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/conv2d_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/conv2d_sets.cpp new file mode 100644 index 00000000000..19eba08ae69 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/conv2d_sets.cpp @@ -0,0 +1,342 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv2d { + +namespace { + +struct Conv2dCaseGroup { + std::vector configs; + std::vector dtypes; + std::vector impls; +}; + +std::vector generate_test_cases( + const std::vector& groups) { + std::vector test_cases; + + for (const auto& group : groups) { + for (const auto& config : group.configs) { + for (auto dtype : group.dtypes) { + for (const auto& impl : group.impls) { + test_cases.push_back(create_conv2d_test_case( + config, dtype, kStorageType, kMemoryLayout, impl)); + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("conv2d", "correctness") { + // Accuracy shapes (small enough for float reference validation) + std::vector accuracy_configs = { + // 3x3 stride=1 pad=1 same-channels (the bottleneck pattern in TinyCNN) + {InputDims(1, 8, 8, 8), + 8, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 8, 8, 8), + 8, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + {InputDims(1, 16, 16, 16), + 16, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // 3x3 stride=2 (downsample) with channel expansion + {InputDims(1, 8, 16, 16), + 16, + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + // 3x3 stride=1 with channel reduction + {InputDims(1, 16, 8, 8), + 8, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + // Non-multiple-of-4 channels + {InputDims(1, 11, 8, 8), + 13, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + // 3-channel input (like RGB stem) + {InputDims(1, 3, 16, 16), + 8, + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + }; + + // Small shapes used to exercise each im2col intermediate-storage variant + // (buffer / texture2d / texture3d) deterministically and independently of + // the device's auto-selection. All dims < kRefDimSizeLimit so both the FP32 + // and FP16 references validate them. The first two shapes keep M (= H_out * + // W_out) tiny, so the im2col intermediate [1, K_total, H_out, W_out] is small + // and the forced texture3d variant never exercises the large-M texture3d + // Z-layout the auto-selector actually falls back to. The third shape (48x48) + // is included specifically so the forced im2col_tex3d variant exercises the + // texture3d layout (K4 along Z, many spatial rows) at a non-trivial M, while + // still keeping every dim under kRefDimSizeLimit for reference validation. + std::vector per_variant_configs = { + // 3x3 s1 p1, channels multiple of 4 + {InputDims(1, 16, 16, 16), + 16, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // Non-multiple-of-4 channels exercise the Cin padding path + {InputDims(1, 11, 12, 12), + 13, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + // Larger spatial extent (M = 48*48 = 2304) exercises the texture3d im2col + // layout [1, K_total, H_out, W_out] with K4 along Z at a non-trivial M, + // while all dims stay < kRefDimSizeLimit so both FP32 and FP16 references + // validate it. C_in=16 keeps K = 3*3*16 = 144 (same as the 16x16 case) + // so FP16 accumulation stays within tolerance. + {InputDims(1, 16, 48, 48), + 16, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + }; + + // Implementation variants exercised for every small ACCU shape: + // "" -> heuristic-routed (should_use_conv2d_im2col picks direct on + // Adreno for small c_out, im2col on Mali) + // "im2col" -> forced im2col/GEMM path + // "direct" -> forced direct sliding-window path (conv2d_direct_impl) + // Including "direct" guarantees the direct shader gets reference-checked on + // BOTH devices — without it, Mali would always route "" to im2col and never + // exercise the direct path. + const std::vector impls = {"", "im2col", "direct"}; + // Forced-storage im2col variants for the per-variant ACCU coverage. + const std::vector forced_storage_impls = { + "im2col_buffer", "im2col_tex2d", "im2col_tex3d"}; + + // FP16 small shapes get a real reference check (gated in + // conv2d_reference_impl); we run both dtypes so we catch correctness + // regressions in either path. Large-K half stays timing-only via the + // reference's PERF-shape throw. + const std::vector accu_dtypes = { + vkapi::kFloat, vkapi::kHalf}; + + const std::vector groups = { + // Accuracy test cases for all impls and both dtypes. + {accuracy_configs, accu_dtypes, impls}, + // Per-variant forced-storage ACCU cases (FP32 and FP16) so all three + // im2col intermediate-storage variants get deterministic, + // device-independent, reference-checked coverage at small K. + {per_variant_configs, accu_dtypes, forced_storage_impls}, + }; + return { + generate_test_cases(groups), + conv2d_reference_impl, + conv2d_flop_calculator, + /*warmup_runs = */ 5, + /*benchmark_runs = */ 20}; +} + +REGISTER_TEST_CASE_SET("conv2d", "performance") { + // TinyCNN depth estimator hotspots (from profiling + // UNTRAINED_TinyCNNDepthEstimatorRealTime_Vulkan.pte). + // Each entry lists (C_in, H, W) -> C_out, all 3x3 stride=1 pad=1 unless + // noted. Together the first 6 entries account for ~89% of all conv time. + std::vector perf_configs = { + // #1: 21.25% — (1,128,36,48)->(1,128,36,48) + {InputDims(1, 128, 36, 48), + 128, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // #2: 20.68% — (1,256,18,24)->(1,256,18,24) + {InputDims(1, 256, 18, 24), + 256, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // #3: 20.01% — (1,64,72,96)->(1,64,72,96) + {InputDims(1, 64, 72, 96), + 64, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // #4: 13.25% — (1,32,144,192)->(1,32,144,192) + {InputDims(1, 32, 144, 192), + 32, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // #5: 6.74% — (1,64,36,48)->(1,64,36,48) + {InputDims(1, 64, 36, 48), + 64, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // #6: 5.90% — (1,32,72,96)->(1,32,72,96) + {InputDims(1, 32, 72, 96), + 32, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // Secondary cases + // 3x3 stride=2 downsample with channel expansion: 1.52% + {InputDims(1, 32, 72, 96), + 128, + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + // 3x3 stride=1 same-shape, smaller spatial: 1.51% + {InputDims(1, 128, 18, 24), + 128, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // 3x3 stride=1, channel reduction + {InputDims(1, 128, 18, 24), + 64, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 64, 36, 48), + 32, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + // 3x3 stride=2 downsample, same channels + {InputDims(1, 32, 72, 96), + 32, + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 64, 36, 48), + 64, + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + // RGB stem + {InputDims(1, 3, 144, 192), + 32, + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + }; + + // Boundary pair straddling the should_use_conv2d_im2col() c_out >= 128 + // routing threshold. Spatial dims are tiny (8x8) so the FP32 float reference + // stays cheap, but c_out = 64 / 128 are both >= kRefDimSizeLimit, so these + // get the PERF label. FP32 PERF cases are still numerically VERIFIED (the + // reference's invalid_argument throw that skips the check only fires for + // half), so both implementations are cross-checked against the float + // reference at the boundary. Run all three impls: at c_out = 64 the heuristic + // ("") picks direct on Adreno / im2col on Mali; at c_out = 128 it picks + // im2col on both — and "direct"/"im2col" force each path regardless, proving + // the two implementations agree at the boundary on either device. + std::vector boundary_configs = { + // c_out = 64 (< 128): below the threshold + {InputDims(1, 16, 8, 8), + 64, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + // c_out = 128 (== 128): at/above the threshold + {InputDims(1, 16, 8, 8), + 128, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + }; + + const std::vector impls = {"", "im2col", "direct"}; + + const std::vector groups = { + // The c_out boundary pair (FP32 only) through all three impls. FP32 PERF + // cases are reference-VERIFIED, so the direct and im2col paths are both + // cross-checked against the float reference at the routing threshold. + {boundary_configs, {vkapi::kFloat}, impls}, + // Performance test cases (float and half) for all impls. + {perf_configs, {vkapi::kFloat, vkapi::kHalf}, impls}, + }; + return { + generate_test_cases(groups), + conv2d_reference_impl, + conv2d_flop_calculator, + /*warmup_runs = */ 5, + /*benchmark_runs = */ 20}; +} + +} // namespace conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/dw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/dw_sets.cpp new file mode 100644 index 00000000000..1b71c730164 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/dw_sets.cpp @@ -0,0 +1,189 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv2d { + +namespace { + +std::vector generate_dw_test_cases( + const std::vector& configs, + const std::vector& dtypes, + const std::vector& impls) { + std::vector test_cases; + + for (const auto& config : configs) { + for (auto dtype : dtypes) { + for (const auto& impl : impls) { + test_cases.push_back(create_conv2d_dw_test_case( + config, dtype, kStorageType, kMemoryLayout, impl)); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("conv2d", "dw_correctness") { + // Accuracy shapes (small enough for float reference validation) + std::vector accuracy_configs = { + {InputDims(1, 8, 16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 8, 16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + {InputDims(1, 8, 16, 16), + KernelSize(5, 5), + Stride(1, 1), + Padding(2, 2), + Dilation(1, 1), + false}, + {InputDims(1, 8, 16, 16), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + // Non-multiple-of-4 channels + {InputDims(1, 11, 16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 3, 16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + }; + + // Accuracy test cases (float only, auto-selected implementation) + return { + generate_dw_test_cases(accuracy_configs, {vkapi::kFloat}, {""}), + conv2d_dw_reference_impl, + conv2d_dw_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("conv2d", "dw_performance") { + // EdgeTAM depthwise shapes (from profiling data) + std::vector perf_configs = { + // Backbone stem and early stages + {InputDims(1, 24, 512, 512), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 48, 256, 256), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 48, 256, 256), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 96, 128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 96, 128, 128), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 192, 64, 64), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 192, 64, 64), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 384, 32, 32), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + // 5x5 kernels + {InputDims(1, 48, 256, 256), + KernelSize(5, 5), + Stride(1, 1), + Padding(2, 2), + Dilation(1, 1), + false}, + {InputDims(1, 96, 128, 128), + KernelSize(5, 5), + Stride(1, 1), + Padding(2, 2), + Dilation(1, 1), + false}, + // FPN/Neck + {InputDims(1, 256, 256, 256), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 256, 128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + }; + + const std::vector impls = { + // Auto-selection (empty impl_selector) + "", + // Force b4x2 variant + "b4x2", + // Force b1x1 variant (only for 3x3 kernels; for 5x5 it falls back + // to default, but we still generate it to test the fallback path) + "b1x1", + }; + + // Performance test cases (float and half) + return { + generate_dw_test_cases( + perf_configs, {vkapi::kFloat, vkapi::kHalf}, impls), + conv2d_dw_reference_impl, + conv2d_dw_flop_calculator}; +} + +} // namespace conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/pw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/pw_sets.cpp new file mode 100644 index 00000000000..afb206ea6e2 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/pw_sets.cpp @@ -0,0 +1,104 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv2d { + +namespace { + +struct Conv2dPwCaseGroup { + std::vector configs; + std::vector dtypes; +}; + +std::vector generate_pw_test_cases( + const std::vector& groups) { + std::vector test_cases; + + for (const auto& group : groups) { + for (const auto& config : group.configs) { + for (auto dtype : group.dtypes) { + test_cases.push_back(create_conv2d_pw_test_case( + config, dtype, kStorageType, kMemoryLayout)); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("conv2d", "pw_correctness") { + // Accuracy shapes (small enough for float reference validation) + std::vector accuracy_configs = { + {1, 16, 32, 8, 8, false}, + {1, 32, 16, 8, 8, false}, + {1, 16, 32, 8, 8, true}, + // Non-multiple-of-4 channels + {1, 13, 27, 8, 8, false}, + {1, 33, 17, 8, 8, false}, + }; + + const std::vector groups = { + // Accuracy test cases (float only) + {accuracy_configs, {vkapi::kFloat}}, + }; + return { + generate_pw_test_cases(groups), + conv2d_pw_reference_impl, + conv2d_pw_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("conv2d", "pw_performance") { + // Accuracy shapes (small enough for float reference validation) that + // create_conv2d_pw_test_case labels PERF, since a channel count exceeds + // kRefDimSizeLimit + std::vector accuracy_configs = { + {1, 48, 96, 16, 16, false}, + {1, 96, 48, 16, 16, false}, + }; + + // EdgeTAM performance shapes + std::vector perf_configs = { + // EdgeTAM backbone stages + {1, 48, 96, 256, 256, false}, + {1, 96, 48, 256, 256, false}, + {1, 96, 192, 128, 128, false}, + {1, 192, 96, 128, 128, false}, + {1, 192, 384, 64, 64, false}, + {1, 384, 192, 64, 64, false}, + {1, 384, 768, 32, 32, false}, + {1, 768, 384, 32, 32, false}, + // EdgeTAM FPN/Neck + {1, 48, 256, 256, 256, false}, + {1, 256, 32, 256, 256, false}, + {1, 96, 256, 128, 128, false}, + {1, 256, 64, 128, 128, false}, + }; + + const std::vector groups = { + // Accuracy test cases (float only) + {accuracy_configs, {vkapi::kFloat}}, + // Performance test cases (float and half) + {perf_configs, {vkapi::kFloat, vkapi::kHalf}}, + }; + return { + generate_pw_test_cases(groups), + conv2d_pw_reference_impl, + conv2d_pw_flop_calculator}; +} + +} // namespace conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.cpp b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.cpp new file mode 100644 index 00000000000..84454d8723d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.cpp @@ -0,0 +1,195 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace embedding_q4gsw { + +TestCase create_test_case(const EmbeddingConfig& config) { + TestCase test_case; + + // Compute the output shape for label: indices_shape + [embed_dim] + std::vector output_shape_for_label = config.indices_shape; + output_shape_for_label.push_back(config.embed_dim); + + // Treat any case that uses the large llama vocab/embed sizes as PERF. + // Otherwise default to ACCU (no PERF distinction is exercised by this op). + bool is_perf = config.vocab_size > 1024 || config.embed_dim > 1024; + std::string prefix = is_perf ? "PERF" : "ACCU"; + + std::string in_dtype = dtype_short(vkapi::kInt); // indices are i32 + std::string out_dtype = dtype_short(config.dtype); + + std::string storage_str = repr_str(config.storage_type, utils::kWidthPacked); + + std::string shape_str = shape_bracket(config.indices_shape) + "x[" + + std::to_string(config.vocab_size) + "," + + std::to_string(config.embed_dim) + "]"; + shape_str += " g" + std::to_string(config.group_size); + if (config.is_linear_weight) { + shape_str += " lw"; + } + std::string suffix; + if (config.scales_dtype == vkapi::kFloat) { + suffix = "[f32_scales]"; + } + std::string name = make_test_label( + prefix, in_dtype, out_dtype, shape_str, storage_str, suffix); + test_case.set_name(name); + test_case.set_operator_name("et_vk.embedding_q4gsw.default"); + test_case.set_shader_filter({}); + + // Weight: [vocab_size, embed_dim / 2] packed uint8 + ValueSpec weight( + {config.vocab_size, config.embed_dim / 2}, + vkapi::kByte, + utils::kBuffer, + utils::kWidthPacked, + DataGenType::RANDINT4); + weight.set_constant(true); + test_case.add_input_spec(weight); + + // Weight scales: [vocab_size, groups_per_row] + int64_t groups_per_row = config.embed_dim / config.group_size; + ValueSpec weight_scales( + {config.vocab_size, groups_per_row}, + config.scales_dtype, + utils::kBuffer, + utils::kWidthPacked, + DataGenType::RANDOM_SCALES); + weight_scales.set_constant(true); + test_case.add_input_spec(weight_scales); + + // Group size: int scalar + ValueSpec group_size_spec(static_cast(config.group_size)); + test_case.add_input_spec(group_size_spec); + + // Indices: [batch, seq_len] int32 + ValueSpec indices( + config.indices_shape, + vkapi::kInt, + config.storage_type, + utils::kWidthPacked, + DataGenType::RANDINT); + + // Clamp indices to valid vocab range + indices.ensure_data_generated(); + for (auto& idx : indices.get_int32_data()) { + idx = std::abs(idx) % config.vocab_size; + } + + test_case.add_input_spec(indices); + + // is_linear_weight: bool scalar + ValueSpec is_linear_weight_spec(config.is_linear_weight); + test_case.add_input_spec(is_linear_weight_spec); + + // Output: indices.shape + [embed_dim] + std::vector output_shape = config.indices_shape; + output_shape.push_back(config.embed_dim); + ValueSpec output( + output_shape, config.dtype, config.storage_type, utils::kWidthPacked); + test_case.add_output_spec(output); + + return test_case; +} + +// CPU reference: unpack 4-bit weights, dequantize, and perform embedding lookup +void embedding_4bit_reference(TestCase& tc) { + auto& weight_spec = tc.inputs()[0]; + auto& scales_spec = tc.inputs()[1]; + int32_t group_size = tc.inputs()[2].get_int_value(); + auto& indices_spec = tc.inputs()[3]; + bool is_linear_weight = tc.inputs()[4].get_bool_value(); + auto& output_spec = tc.outputs()[0]; + + weight_spec.ensure_data_generated(); + scales_spec.ensure_data_generated(); + indices_spec.ensure_data_generated(); + + const auto& weight_data = weight_spec.get_uint8_data(); + const auto& indices_data = indices_spec.get_int32_data(); + + bool scales_are_half = (scales_spec.dtype == vkapi::kHalf); + + int64_t packed_dim = weight_spec.sizes[1]; + int64_t embed_dim = packed_dim * 2; + int64_t groups_per_row = scales_spec.sizes[1]; + + int64_t num_indices = 1; + for (auto s : indices_spec.sizes) { + num_indices *= s; + } + + int64_t total_output = num_indices * embed_dim; + + // Always populate ref_float_data so the caching framework can distribute it + output_spec.get_ref_float_data().resize(total_output); + + bool output_is_half = (output_spec.dtype == vkapi::kHalf); + if (output_is_half) { + output_spec.get_ref_half_data().resize(total_output); + } + + for (int64_t i = 0; i < num_indices; ++i) { + int32_t idx = indices_data[i]; + for (int64_t d = 0; d < embed_dim; ++d) { + int64_t packed_idx = d / 2; + uint8_t packed_byte = weight_data[idx * packed_dim + packed_idx]; + + // Unpack: packed_byte = (even_val + 8) << 4 | (odd_val + 8) + // Even d -> high nibble, odd d -> low nibble + // For linear weight packing, nibble order is swapped + int int4_val; + if (d % 2 == 0) { + if (is_linear_weight) { + int4_val = static_cast(packed_byte & 0xF) - 8; + } else { + int4_val = static_cast(packed_byte >> 4) - 8; + } + } else { + if (is_linear_weight) { + int4_val = static_cast(packed_byte >> 4) - 8; + } else { + int4_val = static_cast(packed_byte & 0xF) - 8; + } + } + + int64_t group_idx = d / group_size; + int64_t scale_idx = idx * groups_per_row + group_idx; + + float scale; + if (scales_are_half) { + uint16_t scale_half = scales_spec.get_half_data()[scale_idx]; + scale = half_to_float(scale_half); + } else { + scale = scales_spec.get_float_data()[scale_idx]; + } + + float result = static_cast(int4_val) * scale; + + // Always store float reference + output_spec.get_ref_float_data()[i * embed_dim + d] = result; + + if (output_is_half) { + output_spec.get_ref_half_data()[i * embed_dim + d] = + float_to_half(result); + } + } + } +} + +} // namespace embedding_q4gsw +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.h b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.h new file mode 100644 index 00000000000..0a2103350a8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.h @@ -0,0 +1,39 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace embedding_q4gsw { + +struct EmbeddingConfig { + int64_t vocab_size; + int64_t embed_dim; + int64_t group_size; + std::vector indices_shape; + std::string test_case_name = "placeholder"; + vkapi::ScalarType dtype = vkapi::kHalf; + vkapi::ScalarType scales_dtype = vkapi::kHalf; + utils::StorageType storage_type = utils::kBuffer; + bool is_linear_weight = false; +}; + +TestCase create_test_case(const EmbeddingConfig& config); + +void embedding_4bit_reference(TestCase& tc); + +} // namespace embedding_q4gsw +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_embedding_q4gsw.cpp b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_embedding_q4gsw.cpp deleted file mode 100644 index 20fcce3c5ca..00000000000 --- a/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_embedding_q4gsw.cpp +++ /dev/null @@ -1,548 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include - -#include -#include - -#include - -using namespace executorch::vulkan::prototyping; - -using namespace vkcompute; - -struct EmbeddingConfig { - int64_t vocab_size; - int64_t embed_dim; - int64_t group_size; - std::vector indices_shape; - std::string test_case_name = "placeholder"; - vkapi::ScalarType dtype = vkapi::kHalf; - vkapi::ScalarType scales_dtype = vkapi::kHalf; - utils::StorageType storage_type = utils::kBuffer; - bool is_linear_weight = false; -}; - -// CPU reference: unpack 4-bit weights, dequantize, and perform embedding lookup -void embedding_4bit_reference(TestCase& tc) { - auto& weight_spec = tc.inputs()[0]; - auto& scales_spec = tc.inputs()[1]; - int32_t group_size = tc.inputs()[2].get_int_value(); - auto& indices_spec = tc.inputs()[3]; - bool is_linear_weight = tc.inputs()[4].get_bool_value(); - auto& output_spec = tc.outputs()[0]; - - weight_spec.ensure_data_generated(); - scales_spec.ensure_data_generated(); - indices_spec.ensure_data_generated(); - - const auto& weight_data = weight_spec.get_uint8_data(); - const auto& indices_data = indices_spec.get_int32_data(); - - bool scales_are_half = (scales_spec.dtype == vkapi::kHalf); - - int64_t packed_dim = weight_spec.sizes[1]; - int64_t embed_dim = packed_dim * 2; - int64_t groups_per_row = scales_spec.sizes[1]; - - int64_t num_indices = 1; - for (auto s : indices_spec.sizes) { - num_indices *= s; - } - - int64_t total_output = num_indices * embed_dim; - - // Always populate ref_float_data so the caching framework can distribute it - output_spec.get_ref_float_data().resize(total_output); - - bool output_is_half = (output_spec.dtype == vkapi::kHalf); - if (output_is_half) { - output_spec.get_ref_half_data().resize(total_output); - } - - for (int64_t i = 0; i < num_indices; ++i) { - int32_t idx = indices_data[i]; - for (int64_t d = 0; d < embed_dim; ++d) { - int64_t packed_idx = d / 2; - uint8_t packed_byte = weight_data[idx * packed_dim + packed_idx]; - - // Unpack: packed_byte = (even_val + 8) << 4 | (odd_val + 8) - // Even d -> high nibble, odd d -> low nibble - // For linear weight packing, nibble order is swapped - int int4_val; - if (d % 2 == 0) { - if (is_linear_weight) { - int4_val = static_cast(packed_byte & 0xF) - 8; - } else { - int4_val = static_cast(packed_byte >> 4) - 8; - } - } else { - if (is_linear_weight) { - int4_val = static_cast(packed_byte >> 4) - 8; - } else { - int4_val = static_cast(packed_byte & 0xF) - 8; - } - } - - int64_t group_idx = d / group_size; - int64_t scale_idx = idx * groups_per_row + group_idx; - - float scale; - if (scales_are_half) { - uint16_t scale_half = scales_spec.get_half_data()[scale_idx]; - scale = half_to_float(scale_half); - } else { - scale = scales_spec.get_float_data()[scale_idx]; - } - - float result = static_cast(int4_val) * scale; - - // Always store float reference - output_spec.get_ref_float_data()[i * embed_dim + d] = result; - - if (output_is_half) { - output_spec.get_ref_half_data()[i * embed_dim + d] = - float_to_half(result); - } - } - } -} - -TestCase create_test_case(const EmbeddingConfig& config) { - TestCase test_case; - - // Compute the output shape for label: indices_shape + [embed_dim] - std::vector output_shape_for_label = config.indices_shape; - output_shape_for_label.push_back(config.embed_dim); - - // Treat any case that uses the large llama vocab/embed sizes as PERF. - // Otherwise default to ACCU (no PERF distinction is exercised by this op). - bool is_perf = config.vocab_size > 1024 || config.embed_dim > 1024; - std::string prefix = is_perf ? "PERF" : "ACCU"; - - std::string in_dtype = dtype_short(vkapi::kInt); // indices are i32 - std::string out_dtype = dtype_short(config.dtype); - - std::string storage_str = repr_str(config.storage_type, utils::kWidthPacked); - - std::string shape_str = shape_bracket(config.indices_shape) + "x[" + - std::to_string(config.vocab_size) + "," + - std::to_string(config.embed_dim) + "]"; - shape_str += " g" + std::to_string(config.group_size); - if (config.is_linear_weight) { - shape_str += " lw"; - } - std::string suffix; - if (config.scales_dtype == vkapi::kFloat) { - suffix = "[f32_scales]"; - } - std::string name = make_test_label( - prefix, in_dtype, out_dtype, shape_str, storage_str, suffix); - test_case.set_name(name); - test_case.set_operator_name("et_vk.embedding_q4gsw.default"); - test_case.set_shader_filter({}); - - // Weight: [vocab_size, embed_dim / 2] packed uint8 - ValueSpec weight( - {config.vocab_size, config.embed_dim / 2}, - vkapi::kByte, - utils::kBuffer, - utils::kWidthPacked, - DataGenType::RANDINT4); - weight.set_constant(true); - test_case.add_input_spec(weight); - - // Weight scales: [vocab_size, groups_per_row] - int64_t groups_per_row = config.embed_dim / config.group_size; - ValueSpec weight_scales( - {config.vocab_size, groups_per_row}, - config.scales_dtype, - utils::kBuffer, - utils::kWidthPacked, - DataGenType::RANDOM_SCALES); - weight_scales.set_constant(true); - test_case.add_input_spec(weight_scales); - - // Group size: int scalar - ValueSpec group_size_spec(static_cast(config.group_size)); - test_case.add_input_spec(group_size_spec); - - // Indices: [batch, seq_len] int32 - ValueSpec indices( - config.indices_shape, - vkapi::kInt, - config.storage_type, - utils::kWidthPacked, - DataGenType::RANDINT); - - // Clamp indices to valid vocab range - indices.ensure_data_generated(); - for (auto& idx : indices.get_int32_data()) { - idx = std::abs(idx) % config.vocab_size; - } - - test_case.add_input_spec(indices); - - // is_linear_weight: bool scalar - ValueSpec is_linear_weight_spec(config.is_linear_weight); - test_case.add_input_spec(is_linear_weight_spec); - - // Output: indices.shape + [embed_dim] - std::vector output_shape = config.indices_shape; - output_shape.push_back(config.embed_dim); - ValueSpec output( - output_shape, config.dtype, config.storage_type, utils::kWidthPacked); - test_case.add_output_spec(output); - - return test_case; -} - -std::vector generate_test_cases() { - std::vector test_cases; - - // --- is_linear_weight = true --- - - // Basic test with linear weight packing - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_linear_weight", - .is_linear_weight = true})); - - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d_linear_weight", - .is_linear_weight = true})); - - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup_linear_weight", - .is_linear_weight = true})); - - // fp32 output with linear weight - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_fp32_linear_weight", - .dtype = vkapi::kFloat, - .is_linear_weight = true})); - - // Texture3D with linear weight - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_texture_linear_weight", - .storage_type = utils::kTexture3D, - .is_linear_weight = true})); - - // Llama 3.2 1B with linear weight packing - test_cases.push_back(create_test_case( - {.vocab_size = 128256, - .embed_dim = 2048, - .group_size = 32, - .indices_shape = {1, 2047}, - .test_case_name = "llama_3_2_1b_prefill_fp32_linear_weight", - .dtype = vkapi::kFloat, - .storage_type = utils::kBuffer, - .is_linear_weight = true})); - - // --- Half scales (default) --- - - // Basic test: small vocab, small embed_dim - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d"})); - - // 2D indices - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d"})); - - // Larger vocab, multiple groups - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup"})); - - // group_size == embed_dim (single group) - test_cases.push_back(create_test_case( - {.vocab_size = 50, - .embed_dim = 64, - .group_size = 64, - .indices_shape = {2, 4}, - .test_case_name = "single_group"})); - - // fp32 output variants (half scales) - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_fp32", - .dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d_fp32", - .dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup_fp32", - .dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 50, - .embed_dim = 64, - .group_size = 64, - .indices_shape = {2, 4}, - .test_case_name = "single_group_fp32", - .dtype = vkapi::kFloat})); - - // Texture3D variants (fp16 output, half scales) - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_texture", - .dtype = vkapi::kHalf, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d_texture", - .dtype = vkapi::kHalf, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup_texture", - .dtype = vkapi::kHalf, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 50, - .embed_dim = 64, - .group_size = 64, - .indices_shape = {2, 4}, - .test_case_name = "single_group_texture", - .dtype = vkapi::kHalf, - .storage_type = utils::kTexture3D})); - - // Texture3D variants (fp32 output, half scales) - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_fp32_texture", - .dtype = vkapi::kFloat, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d_fp32_texture", - .dtype = vkapi::kFloat, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup_fp32_texture", - .dtype = vkapi::kFloat, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 50, - .embed_dim = 64, - .group_size = 64, - .indices_shape = {2, 4}, - .test_case_name = "single_group_fp32_texture", - .dtype = vkapi::kFloat, - .storage_type = utils::kTexture3D})); - - // --- Float scales --- - - // Buffer variants with float scales - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_float_scales", - .dtype = vkapi::kHalf, - .scales_dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d_float_scales", - .dtype = vkapi::kHalf, - .scales_dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup_float_scales", - .dtype = vkapi::kHalf, - .scales_dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_fp32_float_scales", - .dtype = vkapi::kFloat, - .scales_dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d_fp32_float_scales", - .dtype = vkapi::kFloat, - .scales_dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup_fp32_float_scales", - .dtype = vkapi::kFloat, - .scales_dtype = vkapi::kFloat})); - - // Texture3D with float scales - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_float_scales_texture", - .dtype = vkapi::kHalf, - .scales_dtype = vkapi::kFloat, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_fp32_float_scales_texture", - .dtype = vkapi::kFloat, - .scales_dtype = vkapi::kFloat, - .storage_type = utils::kTexture3D})); - - // Llama 3.2 1B prefill configuration (fp32 output, half scales) - test_cases.push_back(create_test_case( - {.vocab_size = 128256, - .embed_dim = 2048, - .group_size = 32, - .indices_shape = {1, 2047}, - .test_case_name = "llama_3_2_1b_prefill_fp32", - .dtype = vkapi::kFloat, - .storage_type = utils::kBuffer})); - - // Llama 3.2 1B prefill configuration (fp32 output, float scales) - test_cases.push_back(create_test_case( - {.vocab_size = 128256, - .embed_dim = 2048, - .group_size = 32, - .indices_shape = {1, 2047}, - .test_case_name = "llama_3_2_1b_prefill_fp32_float_scales", - .dtype = vkapi::kFloat, - .scales_dtype = vkapi::kFloat, - .storage_type = utils::kBuffer})); - - // Llama 3.2 1B prefill configuration (fp16 output, half scales) - test_cases.push_back(create_test_case( - {.vocab_size = 128256, - .embed_dim = 2048, - .group_size = 32, - .indices_shape = {1, 2047}, - .test_case_name = "llama_3_2_1b_prefill_fp16", - .dtype = vkapi::kHalf, - .storage_type = utils::kBuffer})); - - // Llama 3.2 1B prefill configuration (fp16 output, float scales) - test_cases.push_back(create_test_case( - {.vocab_size = 128256, - .embed_dim = 2048, - .group_size = 32, - .indices_shape = {1, 2047}, - .test_case_name = "llama_3_2_1b_prefill_fp16_float_scales", - .dtype = vkapi::kHalf, - .scales_dtype = vkapi::kFloat, - .storage_type = utils::kBuffer})); - - return test_cases; -} - -int main(int argc, char** argv) { - auto results = execute_test_cases( - generate_test_cases, - "embedding_q4gsw", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - embedding_4bit_reference); - - results.print_summary(); - - if (results.get_failed_count() > 0) { - std::cerr << "FAILED: " << results.get_failed_count() << " test(s) failed." - << std::endl; - return 1; - } - - std::cout << "PASSED: All tests passed." << std::endl; - return 0; -} diff --git a/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_sets/embedding_q4gsw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_sets/embedding_q4gsw_sets.cpp new file mode 100644 index 00000000000..614eeb721a7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_sets/embedding_q4gsw_sets.cpp @@ -0,0 +1,328 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace embedding_q4gsw { + +namespace { + +std::vector generate_test_cases( + const std::vector& configs) { + std::vector test_cases; + test_cases.reserve(configs.size()); + for (const auto& config : configs) { + test_cases.push_back(create_test_case(config)); + } + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("embedding_q4gsw", "correctness") { + const std::vector configs = { + // --- is_linear_weight = true --- + + // Basic test with linear weight packing + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_linear_weight", + .is_linear_weight = true}, + + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d_linear_weight", + .is_linear_weight = true}, + + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup_linear_weight", + .is_linear_weight = true}, + + // fp32 output with linear weight + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_fp32_linear_weight", + .dtype = vkapi::kFloat, + .is_linear_weight = true}, + + // Texture3D with linear weight + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_texture_linear_weight", + .storage_type = utils::kTexture3D, + .is_linear_weight = true}, + + // --- Half scales (default) --- + + // Basic test: small vocab, small embed_dim + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d"}, + + // 2D indices + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d"}, + + // Larger vocab, multiple groups + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup"}, + + // group_size == embed_dim (single group) + {.vocab_size = 50, + .embed_dim = 64, + .group_size = 64, + .indices_shape = {2, 4}, + .test_case_name = "single_group"}, + + // fp32 output variants (half scales) + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_fp32", + .dtype = vkapi::kFloat}, + + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d_fp32", + .dtype = vkapi::kFloat}, + + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup_fp32", + .dtype = vkapi::kFloat}, + + {.vocab_size = 50, + .embed_dim = 64, + .group_size = 64, + .indices_shape = {2, 4}, + .test_case_name = "single_group_fp32", + .dtype = vkapi::kFloat}, + + // Texture3D variants (fp16 output, half scales) + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_texture", + .dtype = vkapi::kHalf, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d_texture", + .dtype = vkapi::kHalf, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup_texture", + .dtype = vkapi::kHalf, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 50, + .embed_dim = 64, + .group_size = 64, + .indices_shape = {2, 4}, + .test_case_name = "single_group_texture", + .dtype = vkapi::kHalf, + .storage_type = utils::kTexture3D}, + + // Texture3D variants (fp32 output, half scales) + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_fp32_texture", + .dtype = vkapi::kFloat, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d_fp32_texture", + .dtype = vkapi::kFloat, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup_fp32_texture", + .dtype = vkapi::kFloat, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 50, + .embed_dim = 64, + .group_size = 64, + .indices_shape = {2, 4}, + .test_case_name = "single_group_fp32_texture", + .dtype = vkapi::kFloat, + .storage_type = utils::kTexture3D}, + + // --- Float scales --- + + // Buffer variants with float scales + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_float_scales", + .dtype = vkapi::kHalf, + .scales_dtype = vkapi::kFloat}, + + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d_float_scales", + .dtype = vkapi::kHalf, + .scales_dtype = vkapi::kFloat}, + + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup_float_scales", + .dtype = vkapi::kHalf, + .scales_dtype = vkapi::kFloat}, + + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_fp32_float_scales", + .dtype = vkapi::kFloat, + .scales_dtype = vkapi::kFloat}, + + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d_fp32_float_scales", + .dtype = vkapi::kFloat, + .scales_dtype = vkapi::kFloat}, + + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup_fp32_float_scales", + .dtype = vkapi::kFloat, + .scales_dtype = vkapi::kFloat}, + + // Texture3D with float scales + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_float_scales_texture", + .dtype = vkapi::kHalf, + .scales_dtype = vkapi::kFloat, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_fp32_float_scales_texture", + .dtype = vkapi::kFloat, + .scales_dtype = vkapi::kFloat, + .storage_type = utils::kTexture3D}, + }; + return {generate_test_cases(configs), embedding_4bit_reference}; +} + +REGISTER_TEST_CASE_SET("embedding_q4gsw", "performance") { + const std::vector configs = { + // Llama 3.2 1B with linear weight packing + {.vocab_size = 128256, + .embed_dim = 2048, + .group_size = 32, + .indices_shape = {1, 2047}, + .test_case_name = "llama_3_2_1b_prefill_fp32_linear_weight", + .dtype = vkapi::kFloat, + .storage_type = utils::kBuffer, + .is_linear_weight = true}, + + // Llama 3.2 1B prefill configuration (fp32 output, half scales) + {.vocab_size = 128256, + .embed_dim = 2048, + .group_size = 32, + .indices_shape = {1, 2047}, + .test_case_name = "llama_3_2_1b_prefill_fp32", + .dtype = vkapi::kFloat, + .storage_type = utils::kBuffer}, + + // Llama 3.2 1B prefill configuration (fp32 output, float scales) + {.vocab_size = 128256, + .embed_dim = 2048, + .group_size = 32, + .indices_shape = {1, 2047}, + .test_case_name = "llama_3_2_1b_prefill_fp32_float_scales", + .dtype = vkapi::kFloat, + .scales_dtype = vkapi::kFloat, + .storage_type = utils::kBuffer}, + + // Llama 3.2 1B prefill configuration (fp16 output, half scales) + {.vocab_size = 128256, + .embed_dim = 2048, + .group_size = 32, + .indices_shape = {1, 2047}, + .test_case_name = "llama_3_2_1b_prefill_fp16", + .dtype = vkapi::kHalf, + .storage_type = utils::kBuffer}, + + // Llama 3.2 1B prefill configuration (fp16 output, float scales) + {.vocab_size = 128256, + .embed_dim = 2048, + .group_size = 32, + .indices_shape = {1, 2047}, + .test_case_name = "llama_3_2_1b_prefill_fp16_float_scales", + .dtype = vkapi::kHalf, + .scales_dtype = vkapi::kFloat, + .storage_type = utils::kBuffer}, + }; + return {generate_test_cases(configs), embedding_4bit_reference}; +} + +} // namespace embedding_q4gsw +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/mm/test_mm.cpp b/backends/vulkan/test/ops/benchmark/cases/mm/mm.cpp similarity index 68% rename from backends/vulkan/test/ops/benchmark/cases/mm/test_mm.cpp rename to backends/vulkan/test/ops/benchmark/cases/mm/mm.cpp index 608f0fa369a..0526eac1e42 100644 --- a/backends/vulkan/test/ops/benchmark/cases/mm/test_mm.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/mm/mm.cpp @@ -4,58 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include - -#include -#include - -#include +#include -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 256; +#include +#include +#include -// Return the input data as float regardless of dtype. For kHalf, the -// underlying half_data is converted via half_to_float; for kFloat the -// existing float_data is copied. The CPU reference computes in float and -// the framework compares against (and converts back to) the GPU output. -static std::vector as_float_data(const ValueSpec& spec) { - if (spec.dtype == vkapi::kFloat) { - return spec.get_float_data(); - } - if (spec.dtype == vkapi::kHalf) { - const auto& half_bits = spec.get_half_data(); - std::vector out(half_bits.size()); - for (size_t i = 0; i < half_bits.size(); ++i) { - out[i] = half_to_float(half_bits[i]); - } - return out; - } - throw std::invalid_argument("as_float_data: unsupported dtype"); -} +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace mm { -struct MmConfig { - // mat1: [M, K] or [B, M, K] - // mat2: [K, N] or [B, K, N] - int64_t B; // batch size, 0 for non-batched - int64_t M; - int64_t K; - int64_t N; - bool has_bias; // true for addmm/linear - bool mat2_is_transposed; // true for linear (weight is [N, K]) - bool mat2_is_constant; // true to test prepacked linear path - // "default" routes through aten.{mm,linear,...}.default (production path). - // "coopmat" / "tiled" force-dispatch a specific shader implementation. - std::string impl_selector = "default"; -}; - -struct MmShape { - int64_t B, M, K, N; -}; - -static TestCase create_mm_test_case( +TestCase create_mm_test_case( const MmConfig& config, vkapi::ScalarType dtype, utils::StorageType storage_type, @@ -67,7 +27,7 @@ static TestCase create_mm_test_case( config.N > kRefDimSizeLimit; std::string prefix = is_perf ? "PERF" : "ACCU"; - std::string storage_str = repr_str(storage_type, memory_layout); + std::string storage_str = repr_str(storage_type, utils::kWidthPacked); std::string dtype_str = dtype_short(dtype); // Determine op type string @@ -253,12 +213,73 @@ static TestCase create_mm_test_case( return test_case; } +int64_t mm_flop_calculator(const TestCase& test_case) { + const auto& out_sizes = test_case.outputs()[0].get_tensor_sizes(); + const std::string& op_name = test_case.operator_name(); + + int64_t M, N, K; + + if (op_name == "test_etvk.test_mm.default") { + // mat1[0], mat2[1], impl_selector[2] + auto mat1_sizes = test_case.inputs()[0].get_tensor_sizes(); + M = mat1_sizes[0]; + K = mat1_sizes[1]; + N = out_sizes[1]; + return 2 * M * K * N; + } else if (op_name == "test_etvk.test_bmm.default") { + // mat1[0], mat2[1], impl_selector[2] + auto mat1_sizes = test_case.inputs()[0].get_tensor_sizes(); + int64_t B = mat1_sizes[0]; + M = mat1_sizes[1]; + K = mat1_sizes[2]; + N = out_sizes[2]; + return 2 * B * M * K * N; + } else if (op_name == "test_etvk.test_addmm.default") { + // self[0], mat1[1], mat2[2], beta[3], alpha[4], impl_selector[5] + auto mat1_sizes = test_case.inputs()[1].get_tensor_sizes(); + M = mat1_sizes[0]; + K = mat1_sizes[1]; + N = out_sizes[1]; + return 2 * M * K * N; + } else if (op_name == "test_etvk.test_linear.default") { + // input[0], weight[1], bias[2], impl_selector[3] + auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); + auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); + M = input_sizes[0]; + K = input_sizes[1]; + N = weight_sizes[0]; + return 2 * M * K * N; + } + return 1; +} + +namespace { + +// Return the input data as float regardless of dtype. For kHalf, the +// underlying half_data is converted via half_to_float; for kFloat the +// existing float_data is copied. The CPU reference computes in float and +// the framework compares against (and converts back to) the GPU output. +std::vector as_float_data(const ValueSpec& spec) { + if (spec.dtype == vkapi::kFloat) { + return spec.get_float_data(); + } + if (spec.dtype == vkapi::kHalf) { + const auto& half_bits = spec.get_half_data(); + std::vector out(half_bits.size()); + for (size_t i = 0; i < half_bits.size(); ++i) { + out[i] = half_to_float(half_bits[i]); + } + return out; + } + throw std::invalid_argument("as_float_data: unsupported dtype"); +} + // Reference implementation for mm/bmm // Input layout per test operator: // test_mm/test_bmm: mat1[0], mat2[1], impl_selector[2] // test_addmm: self[0], mat1[1], mat2[2], beta[3], alpha[4], impl_selector[5] // test_linear: input[0], weight[1], bias[2], impl_selector[3] -static void mm_reference_impl(TestCase& test_case) { +void mm_reference_impl(TestCase& test_case) { const std::string& op_name = test_case.operator_name(); ValueSpec& output = test_case.outputs()[0]; auto out_sizes = output.get_tensor_sizes(); @@ -390,189 +411,13 @@ static void mm_reference_impl(TestCase& test_case) { } } -static std::vector generate_mm_test_cases() { - std::vector test_cases; - - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - std::vector shapes = { - // Accuracy shapes (float) - {0, 64, 128, 64}, - {0, 128, 256, 128}, - {0, 32, 64, 256}, - {1, 64, 128, 64}, - {1, 4, 32, 16}, - // Non-multiple-of-4 accuracy shapes (exercises scalar shader fallback) - {0, 57, 131, 43}, - {0, 33, 67, 91}, - {1, 19, 53, 37}, - {0, 64, 128, 47}, // only N unaligned - {0, 64, 47, 128}, // only K unaligned - // Performance shapes (half) - {0, 4096, 1024, 256}, - {0, 4096, 256, 128}, - {0, 4096, 128, 256}, - {1, 4096, 256, 1024}, - {1, 4096, 256, 128}, - {1, 256, 4096, 64}, - {0, 4096, 64, 128}, - }; - - // Coopmat shader requires M%64==0, N%64==0, K%32==0 (no partial-tile or - // K-tail handling). Sweep both "coopmat" and "tiled" force-dispatch variants - // for shapes that satisfy alignment, on buffer storage only (coopmat - // requires buffer outputs). - auto coopmat_shape_eligible = [](const MmShape& s) { - return s.B == 0 && s.M % 64 == 0 && s.N % 64 == 0 && s.K % 32 == 0; - }; - - // The "coopmat" forced-dispatch sweep skips the runtime eligibility check - // (it's intentionally bypassing the gate to exercise the shader). Skip - // generating those cases on adapters that can't actually run the shader, - // or pipeline creation/dispatch will fail. - const auto* adapter = api::context()->adapter_ptr(); - const bool coopmat_runnable = - adapter->supports_cooperative_matrix() && adapter->subgroup_size() == 64; - std::vector coopmat_sweep_selectors = {"tiled"}; - if (coopmat_runnable) { - coopmat_sweep_selectors.push_back("coopmat"); - } - - for (const auto& s : shapes) { - bool is_batched = s.B > 0; - bool is_perf = s.M > kRefDimSizeLimit || s.K > kRefDimSizeLimit || - s.N > kRefDimSizeLimit; - - std::vector dtypes = is_perf - ? std::vector{vkapi::kFloat, vkapi::kHalf} - : std::vector{vkapi::kFloat}; - - MmConfig dynamic_cfg{s.B, s.M, s.K, s.N, false, false, false}; - MmConfig const_cfg{s.B, s.M, s.K, s.N, false, false, true}; - - for (auto dtype : dtypes) { - for (auto st : storage_types) { - test_cases.push_back( - create_mm_test_case(dynamic_cfg, dtype, st, utils::kWidthPacked)); - test_cases.push_back( - create_mm_test_case(const_cfg, dtype, st, utils::kWidthPacked)); - } - - // Coopmat A/B sweep: only on aligned shapes + buffer storage. - if (coopmat_shape_eligible(s)) { - for (const auto& sel : coopmat_sweep_selectors) { - MmConfig dyn = dynamic_cfg; - dyn.impl_selector = sel; - test_cases.push_back(create_mm_test_case( - dyn, dtype, utils::kBuffer, utils::kWidthPacked)); - MmConfig con = const_cfg; - con.impl_selector = sel; - test_cases.push_back(create_mm_test_case( - con, dtype, utils::kBuffer, utils::kWidthPacked)); - } - } - - if (!is_batched) { - MmConfig addmm_cfg{s.B, s.M, s.K, s.N, true, false, false}; - MmConfig addmm_const_cfg{s.B, s.M, s.K, s.N, true, false, true}; - MmConfig linear_cfg{s.B, s.M, s.K, s.N, false, true, false}; - MmConfig linear_bias_cfg{s.B, s.M, s.K, s.N, true, true, false}; - - for (auto st : storage_types) { - test_cases.push_back( - create_mm_test_case(addmm_cfg, dtype, st, utils::kWidthPacked)); - test_cases.push_back(create_mm_test_case( - addmm_const_cfg, dtype, st, utils::kWidthPacked)); - test_cases.push_back( - create_mm_test_case(linear_cfg, dtype, st, utils::kWidthPacked)); - test_cases.push_back(create_mm_test_case( - linear_bias_cfg, dtype, st, utils::kWidthPacked)); - } - - // Coopmat A/B sweep on linear paths too (only aligned shapes). - if (coopmat_shape_eligible(s)) { - for (const auto& sel : coopmat_sweep_selectors) { - MmConfig lin = linear_cfg; - lin.impl_selector = sel; - test_cases.push_back(create_mm_test_case( - lin, dtype, utils::kBuffer, utils::kWidthPacked)); - MmConfig lin_bias = linear_bias_cfg; - lin_bias.impl_selector = sel; - test_cases.push_back(create_mm_test_case( - lin_bias, dtype, utils::kBuffer, utils::kWidthPacked)); - } - } - } - } - } - - return test_cases; -} - -static int64_t mm_flop_calculator(const TestCase& test_case) { - const auto& out_sizes = test_case.outputs()[0].get_tensor_sizes(); - const std::string& op_name = test_case.operator_name(); - - int64_t M, N, K; +} // namespace - if (op_name == "test_etvk.test_mm.default") { - // mat1[0], mat2[1], impl_selector[2] - auto mat1_sizes = test_case.inputs()[0].get_tensor_sizes(); - M = mat1_sizes[0]; - K = mat1_sizes[1]; - N = out_sizes[1]; - return 2 * M * K * N; - } else if (op_name == "test_etvk.test_bmm.default") { - // mat1[0], mat2[1], impl_selector[2] - auto mat1_sizes = test_case.inputs()[0].get_tensor_sizes(); - int64_t B = mat1_sizes[0]; - M = mat1_sizes[1]; - K = mat1_sizes[2]; - N = out_sizes[2]; - return 2 * B * M * K * N; - } else if (op_name == "test_etvk.test_addmm.default") { - // self[0], mat1[1], mat2[2], beta[3], alpha[4], impl_selector[5] - auto mat1_sizes = test_case.inputs()[1].get_tensor_sizes(); - M = mat1_sizes[0]; - K = mat1_sizes[1]; - N = out_sizes[1]; - return 2 * M * K * N; - } else if (op_name == "test_etvk.test_linear.default") { - // input[0], weight[1], bias[2], impl_selector[3] - auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); - auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); - M = input_sizes[0]; - K = input_sizes[1]; - N = weight_sizes[0]; - return 2 * M * K * N; - } - return 1; -} - -static void reference_impl(TestCase& test_case) { +void reference_impl(TestCase& test_case) { mm_reference_impl(test_case); } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Matrix Multiply (mm/bmm/addmm/linear) Benchmark" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_mm_test_cases, - mm_flop_calculator, - "MatMul", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace mm +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/mm/mm.h b/backends/vulkan/test/ops/benchmark/cases/mm/mm.h new file mode 100644 index 00000000000..b4ec8f21795 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/mm/mm.h @@ -0,0 +1,49 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace mm { + +constexpr int64_t kRefDimSizeLimit = 256; + +struct MmConfig { + // mat1: [M, K] or [B, M, K] + // mat2: [K, N] or [B, K, N] + int64_t B; // batch size, 0 for non-batched + int64_t M; + int64_t K; + int64_t N; + bool has_bias; // true for addmm/linear + bool mat2_is_transposed; // true for linear (weight is [N, K]) + bool mat2_is_constant; // true to test prepacked linear path + // "default" routes through aten.{mm,linear,...}.default (production path). + // "coopmat" / "tiled" force-dispatch a specific shader implementation. + std::string impl_selector = "default"; +}; + +TestCase create_mm_test_case( + const MmConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout); + +void reference_impl(TestCase& test_case); + +int64_t mm_flop_calculator(const TestCase& test_case); + +} // namespace mm +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/mm/test_sets/mm_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/mm/test_sets/mm_sets.cpp new file mode 100644 index 00000000000..232e93d326f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/mm/test_sets/mm_sets.cpp @@ -0,0 +1,171 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace mm { + +namespace { + +struct MmShape { + int64_t B, M, K, N; +}; + +// Coopmat shader requires M%64==0, N%64==0, K%32==0 (no partial-tile or K-tail +// handling). The force-dispatch selector sweep runs only for shapes that +// satisfy alignment, on buffer storage only (coopmat requires buffer outputs). +bool coopmat_shape_eligible(const MmShape& s) { + return s.B == 0 && s.M % 64 == 0 && s.N % 64 == 0 && s.K % 32 == 0; +} + +std::vector generate_mm_test_cases( + const std::vector& shapes, + const std::vector& dtypes, + const std::vector& storage_types, + const std::vector& coopmat_sweep_selectors) { + std::vector test_cases; + + // The "coopmat" forced-dispatch sweep skips the runtime eligibility check + // (it's intentionally bypassing the gate to exercise the shader). Skip + // generating those cases on adapters that can't actually run the shader, + // or pipeline creation/dispatch will fail. + const auto* adapter = api::context()->adapter_ptr(); + const bool coopmat_runnable = + adapter->supports_cooperative_matrix() && adapter->subgroup_size() == 64; + std::vector sweep_selectors; + for (const auto& sel : coopmat_sweep_selectors) { + if (sel != "coopmat" || coopmat_runnable) { + sweep_selectors.push_back(sel); + } + } + + for (const auto& s : shapes) { + bool is_batched = s.B > 0; + + MmConfig dynamic_cfg{s.B, s.M, s.K, s.N, false, false, false}; + MmConfig const_cfg{s.B, s.M, s.K, s.N, false, false, true}; + + for (auto dtype : dtypes) { + for (auto st : storage_types) { + test_cases.push_back( + create_mm_test_case(dynamic_cfg, dtype, st, utils::kWidthPacked)); + test_cases.push_back( + create_mm_test_case(const_cfg, dtype, st, utils::kWidthPacked)); + } + + // Coopmat A/B sweep: only on aligned shapes + buffer storage. + if (coopmat_shape_eligible(s)) { + for (const auto& sel : sweep_selectors) { + MmConfig dyn = dynamic_cfg; + dyn.impl_selector = sel; + test_cases.push_back(create_mm_test_case( + dyn, dtype, utils::kBuffer, utils::kWidthPacked)); + MmConfig con = const_cfg; + con.impl_selector = sel; + test_cases.push_back(create_mm_test_case( + con, dtype, utils::kBuffer, utils::kWidthPacked)); + } + } + + if (!is_batched) { + MmConfig addmm_cfg{s.B, s.M, s.K, s.N, true, false, false}; + MmConfig addmm_const_cfg{s.B, s.M, s.K, s.N, true, false, true}; + MmConfig linear_cfg{s.B, s.M, s.K, s.N, false, true, false}; + MmConfig linear_bias_cfg{s.B, s.M, s.K, s.N, true, true, false}; + + for (auto st : storage_types) { + test_cases.push_back( + create_mm_test_case(addmm_cfg, dtype, st, utils::kWidthPacked)); + test_cases.push_back(create_mm_test_case( + addmm_const_cfg, dtype, st, utils::kWidthPacked)); + test_cases.push_back( + create_mm_test_case(linear_cfg, dtype, st, utils::kWidthPacked)); + test_cases.push_back(create_mm_test_case( + linear_bias_cfg, dtype, st, utils::kWidthPacked)); + } + + // Coopmat A/B sweep on linear paths too (only aligned shapes). + if (coopmat_shape_eligible(s)) { + for (const auto& sel : sweep_selectors) { + MmConfig lin = linear_cfg; + lin.impl_selector = sel; + test_cases.push_back(create_mm_test_case( + lin, dtype, utils::kBuffer, utils::kWidthPacked)); + MmConfig lin_bias = linear_bias_cfg; + lin_bias.impl_selector = sel; + test_cases.push_back(create_mm_test_case( + lin_bias, dtype, utils::kBuffer, utils::kWidthPacked)); + } + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("mm", "correctness") { + const std::vector shapes = { + // Accuracy shapes (float) + {0, 64, 128, 64}, + {0, 128, 256, 128}, + {0, 32, 64, 256}, + {1, 64, 128, 64}, + {1, 4, 32, 16}, + // Non-multiple-of-4 accuracy shapes (exercises scalar shader fallback) + {0, 57, 131, 43}, + {0, 33, 67, 91}, + {1, 19, 53, 37}, + {0, 64, 128, 47}, // only N unaligned + {0, 64, 47, 128}, // only K unaligned + }; + const std::vector dtypes = {vkapi::kFloat}; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector coopmat_sweep_selectors = {"tiled", "coopmat"}; + + return { + generate_mm_test_cases( + shapes, dtypes, storage_types, coopmat_sweep_selectors), + reference_impl, + mm_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("mm", "performance") { + const std::vector shapes = { + // Performance shapes (half) + {0, 4096, 1024, 256}, + {0, 4096, 256, 128}, + {0, 4096, 128, 256}, + {1, 4096, 256, 1024}, + {1, 4096, 256, 128}, + {1, 256, 4096, 64}, + {0, 4096, 64, 128}, + }; + const std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector coopmat_sweep_selectors = {"tiled", "coopmat"}; + + return { + generate_mm_test_cases( + shapes, dtypes, storage_types, coopmat_sweep_selectors), + reference_impl, + mm_flop_calculator}; +} + +} // namespace mm +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_q4gsw_linear.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/q4gsw_linear.cpp similarity index 58% rename from backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_q4gsw_linear.cpp rename to backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/q4gsw_linear.cpp index cb06f9692e0..47ab929c38c 100644 --- a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_q4gsw_linear.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/q4gsw_linear.cpp @@ -4,45 +4,19 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include -#include +#include + #include #include -#include +#include +#include +#include #include -#include - -using namespace executorch::vulkan::prototyping; - -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 300; - -// Linear configuration struct -struct LinearConfig { - int64_t M; // Batch size / number of rows in input - int64_t K; // Input features / columns in input, rows in weight - int64_t N; // Output features / columns in weight - int64_t group_size; // Number of input channels per quantization group - bool has_bias = false; - std::string test_case_name = "placeholder"; - std::string op_name = "linear_dq8ca_q4gsw"; -}; - -// Helper function to unpack 4-bit values from uint8 -std::pair unpack_4bit(uint8_t packed) { - // Extract lower 4 bits and upper 4 bits - int8_t lower = packed & 0x0F; - int8_t upper = (packed >> 4) & 0x0F; - - // Subtract 8 from unpacked 4-bit values - lower -= 8; - upper -= 8; - - return std::make_pair(lower, upper); -} +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q4gsw_linear { // Utility function to create a test case from a LinearConfig TestCase create_test_case_from_config( @@ -219,160 +193,264 @@ TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for quantized linear operation (for debugging) -std::vector generate_quantized_linear_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - int M = 8; - int K = 16; - int N = 16; - int group_size = 8; - - LinearConfig config = { - M, // Batch size - K, // Input features - N, // Output features - group_size, // Group size - true, // has_bias - "simple", // test_case_name - }; - - // Test with both storage types and data types for completeness - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - std::vector float_types = {vkapi::kFloat}; - - // Generate test cases for each combination - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, input_dtype)); - } +int64_t quantized_linear_flop_calculator(const TestCase& test_case) { + // Get input and weight dimensions + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); + + int64_t batch_size = input_sizes[0]; + int64_t in_features = input_sizes[1]; + int64_t out_features = output_sizes[1]; + + // Calculate FLOPs for quantized linear operation + // Each output element requires: + // - in_features multiply-accumulate operations + // - Additional operations for quantization/dequantization + int64_t output_elements = batch_size * out_features; + int64_t ops_per_output = in_features; + + // Add quantization overhead (approximate) + // - Unpack 4-bit weight: 1 op per weight element used + // - Dequantize weight: 1 op per weight element used + // - Add bias: 1 op per output element + // - For activation+weight quantized: add input quantization ops + int64_t quantization_ops = ops_per_output * 2 + 1; // Simplified estimate + + int64_t flop = output_elements * (ops_per_output + quantization_ops); + + return flop; +} + +// Create a single test case for the test_fpa_q4gsw_linear.{gemm,gemv} op. +TestCase create_test_case( + const FpaLinearConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage, + int32_t impl_selector, + bool is_gemv) { + TestCase test_case; + + const int64_t M = config.M; + const int64_t K = config.K; + const int64_t N = config.N; + const int64_t group_size = config.group_size; + + const bool is_performance = + (M > kRefDimSizeLimit || K > kRefDimSizeLimit || N > kRefDimSizeLimit); + const std::string prefix = is_performance ? "PERF" : "ACCU"; + + const std::string dtype_str = dtype_short(dtype); + const std::string shape_str = shape_bracket({M, K}) + "x[" + + std::to_string(N) + "," + std::to_string(K) + "] g" + + std::to_string(group_size); + const std::string storage_str = repr_str(storage, utils::kWidthPacked); + std::string suffix = std::string("[") + (is_gemv ? "gemv" : "gemm") + " s" + + std::to_string(impl_selector) + "]"; + suffix += config.has_bias ? " bias" : " no_bias"; + const std::string test_name = make_test_label( + prefix, dtype_str, dtype_str, shape_str, storage_str, suffix); + test_case.set_name(test_name); + + const std::string op_name = is_gemv ? "test_etvk.test_fpa_q4gsw_linear.gemv" + : "test_etvk.test_fpa_q4gsw_linear.gemm"; + test_case.set_operator_name(op_name); + + // Input: [M, K] + ValueSpec input( + {M, K}, dtype, storage, utils::kWidthPacked, DataGenType::RANDINT); + + // Weight: [N, K/2] uint8 packed 4-bit + ValueSpec weight( + {N, K / 2}, + vkapi::kByte, + storage, + utils::kWidthPacked, + DataGenType::RANDINT4); + weight.set_constant(true); + weight.set_int4(true); + + // Scales: [K/gs, N] matching input dtype (the custom op prepacks scales + // using the input tensor's dtype). + ValueSpec scales( + {K / group_size, N}, + dtype, + storage, + utils::kWidthPacked, + DataGenType::RANDOM_SCALES); + scales.set_constant(true); + + // Group size + ValueSpec gs_spec(static_cast(group_size)); + + // Bias + ValueSpec bias( + {N}, + dtype, + storage, + utils::kWidthPacked, + config.has_bias ? DataGenType::RANDOM : DataGenType::ZEROS); + bias.set_constant(true); + if (!config.has_bias) { + bias.set_none(true); } - return test_cases; + // impl_selector as int32 + ValueSpec impl_selector_spec(static_cast(impl_selector)); + + // Output: [M, N] + ValueSpec output( + {M, N}, dtype, storage, utils::kWidthPacked, DataGenType::ZEROS); + + // Tolerance: fp16 outputs use relaxed tolerance to account for f16 + // accumulation / rounding. + float base_tol = 0.05f * (static_cast(K) / 64.0f); + float tol = (dtype == vkapi::kHalf) ? (4.0f * base_tol) : base_tol; + test_case.set_abs_tolerance(tol); + + test_case.add_input_spec(input); + test_case.add_input_spec(weight); + test_case.add_input_spec(scales); + test_case.add_input_spec(gs_spec); + test_case.add_input_spec(bias); + test_case.add_input_spec(impl_selector_spec); + test_case.add_output_spec(output); + + return test_case; } -// Generate test cases for quantized linear operation -std::vector generate_quantized_linear_test_cases() { - std::vector test_cases; - - std::vector configs = { - // // Gemv test cases - // {1, 128, 64, 32}, - // {1, 256, 128, 64}, - // Gemm - {4, 64, 32, 16}, - {4, 128, 64, 32}, - {4, 256, 128, 64}, - {32, 64, 32, 16}, - {32, 128, 64, 32}, - {32, 256, 128, 64}, - {256, 128, 128, 32}, // M=256 > K=128; all dims < kRefDimSizeLimit - - // Coopmat-eligible correctness shapes (M%64==0, N%64==0, K%32==0, - // group_size%32==0). The Buffer+Half variant fires linear_q4gsw_coopmat / - // linear_dq8ca_q4gsw_coopmat and is validated against the CPU reference. - {64, 64, 64, 64}, - {64, 128, 64, 64}, - {64, 256, 128, 128}, - // With bias - {4, 64, 32, 16, true}, - {4, 128, 64, 32, true}, - {32, 128, 64, 32, true}, - // NOTE: coopmat correctness coverage is NOT in this list. The - // coopmat dispatch gate requires M%64==0, N%64==0, K%32==0; the - // smallest qualifying shape (M=64, K=64, N=64) produces enough - // cancellation outputs that fp16 accumulation drift exceeds any - // reasonable tolerance against the fp32 reference. Validating the - // coopmat shader needs a different strategy (e.g. positive-only - // inputs, or simulating fp16 accumulation in the reference). - // A couple of representative performance shapes (coopmat-eligible, - // M % 64 == 0). The full Llama 3.1 8B prefill sweep lived here during - // the study; trimmed to keep this a fast unit test. - {128, 2048, 2048, 128}, - {1024, 4096, 4096, 128}, - }; - - // Test with different storage types and data types - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - for (auto config : configs) { - std::string prefix = - (config.M < kRefDimSizeLimit && config.K < kRefDimSizeLimit && - config.N < kRefDimSizeLimit) - ? "correctness_" - : "performance_"; - std::string generated_test_case_name = prefix + std::to_string(config.M) + - "_" + std::to_string(config.K) + "_" + std::to_string(config.N) + "_g" + - std::to_string(config.group_size); - if (!config.has_bias) { - generated_test_case_name += "_no_bias"; - } +// Custom FLOP calculator: 2 * M * K * N for the linear op itself. +int64_t linear_flop_calculator(const TestCase& test_case) { + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - config.test_case_name = generated_test_case_name; - - // Iterate over both fp32 and fp16 activations so the test covers the - // _float and _half SPIR-V variants of each linear shader. Llama-on-Vulkan - // exports run with backend.vulkan.force_fp16=True, so the _half variants - // are the ones we actually hit in production. - std::vector input_dtypes = {vkapi::kFloat, vkapi::kHalf}; - - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : input_dtypes) { - // Test both activation+weight quantized and weight only quantized, but - // only if the current device supports int8 dot product - if (vkcompute::api::context() - ->adapter_ptr() - ->supports_int8_dot_product()) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, input_dtype)); - } + int64_t M = input_sizes[0]; + int64_t K = input_sizes[1]; + int64_t N = output_sizes[1]; + return 2 * M * K * N; +} - LinearConfig wo_quant_config = config; - wo_quant_config.op_name = "linear_q4gsw"; - test_cases.push_back(create_test_case_from_config( - wo_quant_config, storage_type, input_dtype)); - } - } +bool is_dq8ca(const std::string& op) { + return op.find("dq8ca") != std::string::npos; +} +bool is_4bit(const std::string& op) { + return op.find("q4gsw") != std::string::npos; +} + +// Build one test case for the given op at (storage, half dtype), no bias. +TestCase make_case(const CoopmatLinearConfig& cfg, utils::StorageType storage) { + const vkapi::ScalarType dt = vkapi::kHalf; + TestCase tc; + const std::string storage_str = + (storage == utils::kTexture3D) ? "Texture3D" : "Buffer"; + tc.set_name( + cfg.op_name + "_M" + std::to_string(cfg.M) + "_K" + + std::to_string(cfg.K) + "_N" + std::to_string(cfg.N) + "_" + storage_str); + tc.set_operator_name("et_vk." + cfg.op_name + ".default"); + + ValueSpec input( + {cfg.M, cfg.K}, dt, storage, utils::kWidthPacked, DataGenType::RANDINT); + + // dynamic per-row activation scale/zp (dq8ca only) + ValueSpec input_scale( + {1, cfg.M}, dt, storage, utils::kWidthPacked, DataGenType::RANDOM_SCALES); + input_scale.set_constant(true); + ValueSpec input_zp( + {1, cfg.M}, + vkapi::kChar, + storage, + utils::kWidthPacked, + DataGenType::RANDINT); + input_zp.set_constant(true); + + // weight + scales + sums depend on 4-bit vs 8-bit + const bool four = is_4bit(cfg.op_name); + ValueSpec qweight( + four ? std::vector{cfg.N, cfg.K / 2} + : std::vector{cfg.N, cfg.K}, + four ? vkapi::kByte : vkapi::kChar, + storage, + utils::kWidthPacked, + four ? DataGenType::RANDINT4 : DataGenType::RANDINT8); + qweight.set_constant(true); + if (four) { + qweight.set_int4(true); } - // The narrow 2-row output tile (`tiledm2`) is selected only on Mali, which - // no CI runner has, and the weight-only tests above exercise a different - // implementation. Force the variant so it is checked against the CPU - // reference everywhere. M of 2, 3 and 5 covers an exact multiple of the - // tile, one partial tile, and two tiles plus a remainder, which is where a - // wrong row bound or a missing bounds check shows up. - for (int64_t M : {2, 3, 5}) { - for (bool has_bias : {true, false}) { - LinearConfig config; - config.M = M; - config.K = 64; - config.N = 32; - config.group_size = 32; - config.has_bias = has_bias; - config.op_name = "linear_dq8ca_q4gsw"; - config.test_case_name = "correctness_tiledm2_M" + std::to_string(M) + - (has_bias ? "" : "_no_bias"); - - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : {vkapi::kFloat, vkapi::kHalf}) { - // No int8-dot-product gate here on purpose: forcing the - // fallback makes this path reachable on any GPU, which is the - // point of the variant being hard to reach otherwise. - TestCase tc = - create_test_case_from_config(config, storage_type, input_dtype); - tc.set_name(tc.name() + " [tiledm2]"); - tc.set_force_narrow_int4_tile(true); - test_cases.push_back(tc); - } - } - } + std::vector scales_size = four + ? std::vector{cfg.K / cfg.group_size, cfg.N} + : std::vector{cfg.N}; + ValueSpec weight_scales( + scales_size, + dt, + storage, + utils::kWidthPacked, + DataGenType::RANDOM_SCALES); + weight_scales.set_constant(true); + + ValueSpec weight_sums( + scales_size, + vkapi::kInt, + storage, + utils::kWidthPacked, + DataGenType::ZEROS); + weight_sums.set_constant(true); + if (four) { + compute_weight_sums_4bit_grouped( + weight_sums, qweight, cfg.K / cfg.group_size, cfg.N, cfg.group_size); + } else { + compute_weight_sums(weight_sums, qweight, cfg.N, cfg.K); } - return test_cases; + ValueSpec group_size_spec(static_cast(cfg.group_size)); + + ValueSpec bias({cfg.N}, dt, storage, utils::kWidthPacked, DataGenType::ZEROS); + bias.set_constant(true); + bias.set_none(true); + + ValueSpec output( + {cfg.M, cfg.N}, dt, storage, utils::kWidthPacked, DataGenType::ZEROS); + + // assemble per op signature + if (cfg.op_name == "linear_q4gsw") { + tc.add_input_spec(input); + tc.add_input_spec(qweight); + tc.add_input_spec(weight_scales); + tc.add_input_spec(group_size_spec); + tc.add_input_spec(bias); + } else if (cfg.op_name == "linear_dq8ca_q4gsw") { + tc.add_input_spec(input); + tc.add_input_spec(input_scale); + tc.add_input_spec(input_zp); + tc.add_input_spec(qweight); + tc.add_input_spec(weight_sums); + tc.add_input_spec(weight_scales); + tc.add_input_spec(group_size_spec); + tc.add_input_spec(bias); + } + tc.add_output_spec(output); + return tc; +} + +int64_t flop_calc(const TestCase& tc) { + const auto& in = tc.inputs()[0].get_tensor_sizes(); + const auto& out = tc.outputs()[0].get_tensor_sizes(); + const int64_t M = in[0], K = in[1], N = out[1]; + return 2 * M * N * K; // MAC = 2 flops +} + +namespace { + +// Helper function to unpack 4-bit values from uint8 +std::pair unpack_4bit(uint8_t packed) { + // Extract lower 4 bits and upper 4 bits + int8_t lower = packed & 0x0F; + int8_t upper = (packed >> 4) & 0x0F; + + // Subtract 8 from unpacked 4-bit values + lower -= 8; + upper -= 8; + + return std::make_pair(lower, upper); } // Reference implementation for 4-bit group symmetric weight quantized linear @@ -576,25 +654,16 @@ void linear_dq8ca_q4gsw_reference_impl(TestCase& test_case) { } } -// Reference for the weight-only fallback that linear_dq8ca_q4gsw takes when -// the device has no int8 dot product (see quantized_linear_impl). The op keeps -// its dq8ca signature, so the activation scale/zp and weight sums are present -// but unused: the GPU multiplies float activations by dequantized weights, and -// so does this. void linear_dq8ca_q4gsw_weight_only_reference_impl(TestCase& test_case) { const ValueSpec& input_spec = test_case.inputs()[0]; - // inputs()[1] input_scale and inputs()[2] input_zp are unused on this path const ValueSpec& weight_spec = test_case.inputs()[3]; - // inputs()[4] weight_sums is unused on this path const ValueSpec& weight_scales_spec = test_case.inputs()[5]; const ValueSpec& group_size_spec = test_case.inputs()[6]; const ValueSpec& bias_spec = test_case.inputs()[7]; - ValueSpec& output_spec = test_case.outputs()[0]; - auto input_sizes = input_spec.get_tensor_sizes(); - auto output_sizes = output_spec.get_tensor_sizes(); - + const auto input_sizes = input_spec.get_tensor_sizes(); + const auto output_sizes = output_spec.get_tensor_sizes(); const int64_t batch_size = input_sizes[0]; const int64_t in_features = input_sizes[1]; const int64_t out_features = output_sizes[1]; @@ -609,7 +678,7 @@ void linear_dq8ca_q4gsw_weight_only_reference_impl(TestCase& test_case) { throw std::invalid_argument("Unsupported dtype"); } - auto& weight_data = weight_spec.get_uint8_data(); + const auto& weight_data = weight_spec.get_uint8_data(); auto& ref_data = output_spec.get_ref_float_data(); ref_data.resize(batch_size * out_features); @@ -618,18 +687,14 @@ void linear_dq8ca_q4gsw_weight_only_reference_impl(TestCase& test_case) { float sum = 0.0f; for (int64_t in_f = 0; in_f < in_features; ++in_f) { const float input_val = input_spec.get_element(b * in_features + in_f); - const int64_t group_idx = in_f / group_size; const int64_t scales_idx = group_idx * out_features + out_f; - const int64_t weight_idx = out_f * (in_features / 2) + (in_f / 2); - const uint8_t packed_weight = weight_data[weight_idx]; - const auto unpacked = unpack_4bit(packed_weight); - const int8_t weight_4bit = + const auto unpacked = unpack_4bit(weight_data[weight_idx]); + const int8_t weight = (in_f % 2 == 0) ? unpacked.first : unpacked.second; - - const float weight_scale = weight_scales_spec.get_element(scales_idx); - sum += input_val * static_cast(weight_4bit) * weight_scale; + sum += input_val * static_cast(weight) * + weight_scales_spec.get_element(scales_idx); } if (!bias_spec.is_none()) { sum += bias_spec.get_element(out_f); @@ -639,9 +704,9 @@ void linear_dq8ca_q4gsw_weight_only_reference_impl(TestCase& test_case) { } } +} // namespace + void reference_impl(TestCase& test_case) { - // The forced-narrow-tile cases run the weight-only fallback regardless of - // what the device supports, so they need the fallback's reference. if (test_case.name().find("tiledm2") != std::string::npos) { linear_dq8ca_q4gsw_weight_only_reference_impl(test_case); } else if (test_case.operator_name().find("dq8ca") != std::string::npos) { @@ -651,56 +716,185 @@ void reference_impl(TestCase& test_case) { } } -int64_t quantized_linear_flop_calculator(const TestCase& test_case) { - // Get input and weight dimensions - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); +namespace { - int64_t batch_size = input_sizes[0]; - int64_t in_features = input_sizes[1]; - int64_t out_features = output_sizes[1]; +// Convert a ValueSpec's input data (float or half) into a flat +// std::vector for use in the reference implementation. +std::vector input_to_float_vec(const ValueSpec& spec) { + if (spec.dtype == vkapi::kFloat) { + return spec.get_float_data(); + } + if (spec.dtype == vkapi::kHalf) { + const auto& half_data = spec.get_half_data(); + std::vector out(half_data.size()); + for (size_t i = 0; i < half_data.size(); ++i) { + out[i] = half_to_float(half_data[i]); + } + return out; + } + throw std::invalid_argument( + "Reference implementation supports only float/half input dtypes."); +} - // Calculate FLOPs for quantized linear operation - // Each output element requires: - // - in_features multiply-accumulate operations - // - Additional operations for quantization/dequantization - int64_t output_elements = batch_size * out_features; - int64_t ops_per_output = in_features; +// Reference implementation: simple dequant + fp32 GEMM. Only runs for +// small shapes (gate on kRefDimSizeLimit). +void fpa_linear_q4gsw_reference_impl(TestCase& test_case) { + int32_t idx = 0; + const ValueSpec& input_spec = test_case.inputs()[idx++]; + const ValueSpec& weight_spec = test_case.inputs()[idx++]; + const ValueSpec& scales_spec = test_case.inputs()[idx++]; + const ValueSpec& gs_spec = test_case.inputs()[idx++]; + const ValueSpec& bias_spec = test_case.inputs()[idx++]; + // impl_selector is not used in the reference impl + ++idx; - // Add quantization overhead (approximate) - // - Unpack 4-bit weight: 1 op per weight element used - // - Dequantize weight: 1 op per weight element used - // - Add bias: 1 op per output element - // - For activation+weight quantized: add input quantization ops - int64_t quantization_ops = ops_per_output * 2 + 1; // Simplified estimate + ValueSpec& output_spec = test_case.outputs()[0]; - int64_t flop = output_elements * (ops_per_output + quantization_ops); + auto input_sizes = input_spec.get_tensor_sizes(); + auto output_sizes = output_spec.get_tensor_sizes(); - return flop; + int64_t M = input_sizes[0]; + int64_t K = input_sizes[1]; + int64_t N = output_sizes[1]; + int64_t group_size = gs_spec.get_int_value(); + + if (M > kRefDimSizeLimit || K > kRefDimSizeLimit || N > kRefDimSizeLimit) { + throw std::invalid_argument( + "Dimensions exceed limit for reference implementation."); + } + + std::vector input_data = input_to_float_vec(input_spec); + auto& weight_data = weight_spec.get_uint8_data(); + std::vector scales_data = input_to_float_vec(scales_spec); + std::vector bias_data; + if (!bias_spec.is_none()) { + bias_data = input_to_float_vec(bias_spec); + } + + int64_t num_output_elements = M * N; + auto& ref_data = output_spec.get_ref_float_data(); + ref_data.resize(num_output_elements); + + for (int64_t m = 0; m < M; ++m) { + for (int64_t n = 0; n < N; ++n) { + float sum = 0.0f; + for (int64_t k = 0; k < K; ++k) { + float input_val = input_data[m * K + k]; + + int64_t weight_idx = n * (K / 2) + (k / 2); + uint8_t packed = weight_data[weight_idx]; + int8_t nibble = (k % 2 == 0) + ? static_cast(packed & 0x0F) - 8 + : static_cast((packed >> 4) & 0x0F) - 8; + + int64_t group_idx = k / group_size; + float scale = scales_data[group_idx * N + n]; + + sum += input_val * static_cast(nibble) * scale; + } + if (!bias_spec.is_none()) { + sum += bias_data[n]; + } + ref_data[m * N + n] = sum; + } + } +} + +} // namespace + +void fpa_reference_impl(TestCase& test_case) { + fpa_linear_q4gsw_reference_impl(test_case); } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "4-bit Group Symmetric Weight Quantized Linear Operation Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - // Execute easy test cases using the new framework with custom FLOP calculator - auto results = execute_test_cases( - generate_quantized_linear_test_cases, - quantized_linear_flop_calculator, - "QuantizedLinearQ4GSW", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; +namespace { + +// ---- correctness reference for all four ops; oversized shapes (the perf +// cases) throw -> framework marks them SKIPPED. For dq8ca the activation +// quant round-trip (round(x/scale)+zp) is mirrored in fp32; this is exact +// (not just close) for the coopmat_correctness data, which uses scale=1/16, +// zp=0 and activations that are multiples of 1/16, so fp16-vs-fp32 +// divergence cannot occur. ---- +std::vector as_f(const ValueSpec& s) { + if (s.dtype == vkapi::kFloat) { + return s.get_float_data(); + } + const auto& h = s.get_half_data(); + std::vector o(h.size()); + for (size_t i = 0; i < h.size(); ++i) { + o[i] = half_to_float(h[i]); + } + return o; +} + +} // namespace + +void bench_reference(TestCase& tc) { + const std::string op = tc.operator_name(); + const bool dq8ca = op.find("dq8ca") != std::string::npos; + const bool four = op.find("q4gsw") != std::string::npos; + const ValueSpec& in = tc.inputs()[0]; + ValueSpec& out = tc.outputs()[0]; + const auto is = in.get_tensor_sizes(); + const int64_t M = is[0], K = is[1]; + const int64_t N = out.get_tensor_sizes()[1]; + if (M > 256 || K > 256 || N > 256) { + throw std::invalid_argument("ref: too big"); + } + // input layouts: weight-only = {in, w, w_scales, [group], bias}; + // dq8ca = {in, in_scale, in_zp, w, w_sums, w_scales, [group], bias} + const ValueSpec& w = tc.inputs()[dq8ca ? 3 : 1]; + const ValueSpec& sc = tc.inputs()[dq8ca ? 5 : 2]; + const int64_t group = four ? tc.inputs()[dq8ca ? 6 : 3].get_int_value() : K; + const ValueSpec& bias = tc.inputs()[dq8ca ? (four ? 7 : 6) : (four ? 4 : 3)]; + const bool has_bias = !bias.is_none(); + + const std::vector inf = as_f(in); + const std::vector scf = as_f(sc); + const std::vector bf = has_bias ? as_f(bias) : std::vector(); + const std::vector in_scale = + dq8ca ? as_f(tc.inputs()[1]) : std::vector(); + const std::vector& in_zp = + dq8ca ? tc.inputs()[2].get_int8_data() : std::vector(); + const std::vector& w4 = + four ? w.get_uint8_data() : std::vector(); // [N, K/2] nibbles + const std::vector& w8 = + four ? std::vector() : w.get_int8_data(); // [N, K] + + auto& ref = out.get_ref_float_data(); + ref.resize(M * N); + for (int64_t m = 0; m < M; ++m) { + const float s_in = dq8ca ? in_scale[m] : 1.0f; + const int zp = dq8ca ? int(in_zp[m]) : 0; + for (int64_t n = 0; n < N; ++n) { + float acc = 0.0f; + for (int64_t k = 0; k < K; ++k) { + float a = inf[m * K + k]; + if (dq8ca) { + float q = std::round(a / s_in) + float(zp); + q = std::min(std::max(q, -128.0f), 127.0f); + a = q - float(zp); + } + int wv; + if (four) { + const uint8_t byte = w4[n * (K / 2) + k / 2]; + const int nib = (k & 1) ? ((byte >> 4) & 0xF) : (byte & 0xF); + wv = nib - 8; + } else { + wv = w8[n * K + k]; + } + const float w_scale = four ? scf[(k / group) * N + n] : scf[n]; + acc += a * float(wv) * w_scale; + } + float r = dq8ca ? acc * s_in : acc; + if (has_bias) { + r += bf[n]; + } + ref[m * N + n] = r; + } + } } + +} // namespace q4gsw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/q4gsw_linear.h b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/q4gsw_linear.h new file mode 100644 index 00000000000..5ee95032801 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/q4gsw_linear.h @@ -0,0 +1,130 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q4gsw_linear { + +constexpr int64_t kRefDimSizeLimit = 300; + +// Linear configuration struct +struct LinearConfig { + int64_t M; // Batch size / number of rows in input + int64_t K; // Input features / columns in input, rows in weight + int64_t N; // Output features / columns in weight + int64_t group_size; // Number of input channels per quantization group + bool has_bias = false; + std::string test_case_name = "placeholder"; + std::string op_name = "linear_dq8ca_q4gsw"; +}; + +TestCase create_test_case_from_config( + const LinearConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype); + +void reference_impl(TestCase& test_case); + +int64_t quantized_linear_flop_calculator(const TestCase& test_case); + +// FPA Q4GSW Linear A/B benchmark binary. +// +// Each generated test case has an `impl_selector` arg routed to the test +// op `test_etvk.test_fpa_q4gsw_linear.{gemm,gemv}` in TestFpaQ4gswLinear.cpp: +// +// GEMM (is_gemv=false): +// 0 -> PROD (et_vk.q4gsw_linear.default; dtype-based +// picker) 1 -> GEMM_W_4X8 (forced non-tin GEMM, nc buffer +// weight) 2 -> GEMM_TIN_W_4X8 (forced tin GEMM, nc buffer +// weight) 3 -> LEGACY (et_vk.linear_q4gsw.default +// legacy shaders) +// +// GEMV (is_gemv=true): +// 0 -> PROD (et_vk.q4gsw_linear.default; +// dtype-based picker) 1 -> GEMV_W_4X8 (forced gemv with +// subgroup broadcast) 2 -> GEMV_W_4X8_NOSG (forced gemv +// without subgroup broadcast) 3 -> LEGACY (et_vk.linear_q4gsw.default +// legacy shaders) 13 -> GEMV_COOP_W_4X8_NC_BUFFER (coop GEMV reusing the +// production +// nc-buffer prepack — same payload as +// W_4X8 GEMM/TIN GEMM/sg-GEMV; +// == g1w64 decomposition) +// 14 -> GEMV_COOP_..._G1W64 (force NUM_GROUPS=1, +// WORKERS_PER_GROUP=64) 15 -> GEMV_COOP_..._G4W16 (force +// NUM_GROUPS=4, WORKERS_PER_GROUP=16) 16 -> GEMV_COOP_..._G8W8 (force +// NUM_GROUPS=8, WORKERS_PER_GROUP=8) +// +// Selectors 14-16 pin the coop nc-buffer GEMV to an explicit reduction +// decomposition regardless of N. The production picker (pick_coop_variant_for_N +// in Q4gswLinear.cpp) only chooses g4w16 / g8w8 at PERF-sized N where the +// reference impl is skipped; these forced selectors give g4w16 / g8w8 numeric +// (ACCU) coverage at small N. Production picker behavior is unchanged. +// +// Selector 3 (LEGACY) is the in-prod q4gsw linear path. It uses a different +// prepack (pack_q4_linear_weight) and shader family +// (linear_q4gsw_tiled_* / linear_q4gsw_coop_*); the framework's per-shader +// timing breakdown will pick those up automatically. + +// Linear configuration struct. +struct FpaLinearConfig { + int64_t M; + int64_t K; + int64_t N; + int64_t group_size; + bool has_bias = false; +}; + +TestCase create_test_case( + const FpaLinearConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage, + int32_t impl_selector, + bool is_gemv); + +void fpa_reference_impl(TestCase& test_case); + +int64_t linear_flop_calculator(const TestCase& test_case); + +// Consolidated coopmat-vs-tiled microbenchmark for the two int4-quantized +// linear types at Llama 3.1 8B prefill shapes: +// 4w = linear_q4gsw (weight-only int4) +// 8da4w = linear_dq8ca_q4gsw (dyn-act int8 x int4 weight) +// +// Baseline (tiled) is selected by Texture3D+Half output storage; coopmat is +// selected by Buffer+Half (the runtime gate in QuantizedLinear.cpp picks the +// _coopmat shader when M%64==0, N%64==0, K%32==0, subgroup==64). The CPU +// reference only runs for the small shapes in coopmat_correctness. + +struct CoopmatLinearConfig { + int64_t M; + int64_t K; + int64_t N; + int64_t group_size; // only meaningful for 4-bit + std::string op_name; +}; + +bool is_dq8ca(const std::string& op); + +bool is_4bit(const std::string& op); + +TestCase make_case(const CoopmatLinearConfig& cfg, utils::StorageType storage); + +void bench_reference(TestCase& tc); + +int64_t flop_calc(const TestCase& tc); + +} // namespace q4gsw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_coopmat_linear_bench.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_coopmat_linear_bench.cpp deleted file mode 100644 index 8cd9342f15e..00000000000 --- a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_coopmat_linear_bench.cpp +++ /dev/null @@ -1,406 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -// Consolidated coopmat-vs-tiled microbenchmark for the two int4-quantized -// linear types at Llama 3.1 8B prefill shapes: -// 4w = linear_q4gsw (weight-only int4) -// 8da4w = linear_dq8ca_q4gsw (dyn-act int8 x int4 weight) -// -// Baseline (tiled) is selected by Texture3D+Half output storage; coopmat is -// selected by Buffer+Half (the runtime gate in QuantizedLinear.cpp picks the -// _coopmat shader when M%64==0, N%64==0, K%32==0, subgroup==64). Perf-only: -// no CPU reference is run (correctness is covered by the per-op test_*_linear -// benches at small shapes). - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -struct LinearConfig { - int64_t M; - int64_t K; - int64_t N; - int64_t group_size; // only meaningful for 4-bit - std::string op_name; -}; - -static bool is_dq8ca(const std::string& op) { - return op.find("dq8ca") != std::string::npos; -} -static bool is_4bit(const std::string& op) { - return op.find("q4gsw") != std::string::npos; -} - -// Build one test case for the given op at (storage, half dtype), no bias. -static TestCase make_case(const LinearConfig& cfg, utils::StorageType storage) { - const vkapi::ScalarType dt = vkapi::kHalf; - TestCase tc; - const std::string storage_str = - (storage == utils::kTexture3D) ? "Texture3D" : "Buffer"; - tc.set_name( - cfg.op_name + "_M" + std::to_string(cfg.M) + "_K" + - std::to_string(cfg.K) + "_N" + std::to_string(cfg.N) + "_" + storage_str); - tc.set_operator_name("et_vk." + cfg.op_name + ".default"); - - ValueSpec input( - {cfg.M, cfg.K}, dt, storage, utils::kWidthPacked, DataGenType::RANDINT); - - // dynamic per-row activation scale/zp (dq8ca only) - ValueSpec input_scale( - {1, cfg.M}, dt, storage, utils::kWidthPacked, DataGenType::RANDOM_SCALES); - input_scale.set_constant(true); - ValueSpec input_zp( - {1, cfg.M}, - vkapi::kChar, - storage, - utils::kWidthPacked, - DataGenType::RANDINT); - input_zp.set_constant(true); - - // weight + scales + sums depend on 4-bit vs 8-bit - const bool four = is_4bit(cfg.op_name); - ValueSpec qweight( - four ? std::vector{cfg.N, cfg.K / 2} - : std::vector{cfg.N, cfg.K}, - four ? vkapi::kByte : vkapi::kChar, - storage, - utils::kWidthPacked, - four ? DataGenType::RANDINT4 : DataGenType::RANDINT8); - qweight.set_constant(true); - if (four) { - qweight.set_int4(true); - } - - std::vector scales_size = four - ? std::vector{cfg.K / cfg.group_size, cfg.N} - : std::vector{cfg.N}; - ValueSpec weight_scales( - scales_size, - dt, - storage, - utils::kWidthPacked, - DataGenType::RANDOM_SCALES); - weight_scales.set_constant(true); - - ValueSpec weight_sums( - scales_size, - vkapi::kInt, - storage, - utils::kWidthPacked, - DataGenType::ZEROS); - weight_sums.set_constant(true); - if (four) { - compute_weight_sums_4bit_grouped( - weight_sums, qweight, cfg.K / cfg.group_size, cfg.N, cfg.group_size); - } else { - compute_weight_sums(weight_sums, qweight, cfg.N, cfg.K); - } - - ValueSpec group_size_spec(static_cast(cfg.group_size)); - - ValueSpec bias({cfg.N}, dt, storage, utils::kWidthPacked, DataGenType::ZEROS); - bias.set_constant(true); - bias.set_none(true); - - ValueSpec output( - {cfg.M, cfg.N}, dt, storage, utils::kWidthPacked, DataGenType::ZEROS); - - // assemble per op signature - if (cfg.op_name == "linear_q4gsw") { - tc.add_input_spec(input); - tc.add_input_spec(qweight); - tc.add_input_spec(weight_scales); - tc.add_input_spec(group_size_spec); - tc.add_input_spec(bias); - } else if (cfg.op_name == "linear_dq8ca_q4gsw") { - tc.add_input_spec(input); - tc.add_input_spec(input_scale); - tc.add_input_spec(input_zp); - tc.add_input_spec(qweight); - tc.add_input_spec(weight_sums); - tc.add_input_spec(weight_scales); - tc.add_input_spec(group_size_spec); - tc.add_input_spec(bias); - } - tc.add_output_spec(output); - return tc; -} - -// ---- correctness reference for all four ops; oversized shapes (the perf -// cases) throw -> framework marks them SKIPPED. For dq8ca the activation -// quant round-trip (round(x/scale)+zp) is mirrored in fp32; this is exact -// (not just close) for the correctness data below, which uses scale=1/16, -// zp=0 and activations that are multiples of 1/16, so fp16-vs-fp32 -// divergence cannot occur. ---- -static std::vector as_f(const ValueSpec& s) { - if (s.dtype == vkapi::kFloat) { - return s.get_float_data(); - } - const auto& h = s.get_half_data(); - std::vector o(h.size()); - for (size_t i = 0; i < h.size(); ++i) { - o[i] = half_to_float(h[i]); - } - return o; -} -static void bench_reference(TestCase& tc) { - const std::string op = tc.operator_name(); - const bool dq8ca = op.find("dq8ca") != std::string::npos; - const bool four = op.find("q4gsw") != std::string::npos; - const ValueSpec& in = tc.inputs()[0]; - ValueSpec& out = tc.outputs()[0]; - const auto is = in.get_tensor_sizes(); - const int64_t M = is[0], K = is[1]; - const int64_t N = out.get_tensor_sizes()[1]; - if (M > 256 || K > 256 || N > 256) { - throw std::invalid_argument("ref: too big"); - } - // input layouts: weight-only = {in, w, w_scales, [group], bias}; - // dq8ca = {in, in_scale, in_zp, w, w_sums, w_scales, [group], bias} - const ValueSpec& w = tc.inputs()[dq8ca ? 3 : 1]; - const ValueSpec& sc = tc.inputs()[dq8ca ? 5 : 2]; - const int64_t group = four ? tc.inputs()[dq8ca ? 6 : 3].get_int_value() : K; - const ValueSpec& bias = tc.inputs()[dq8ca ? (four ? 7 : 6) : (four ? 4 : 3)]; - const bool has_bias = !bias.is_none(); - - const std::vector inf = as_f(in); - const std::vector scf = as_f(sc); - const std::vector bf = has_bias ? as_f(bias) : std::vector(); - const std::vector in_scale = - dq8ca ? as_f(tc.inputs()[1]) : std::vector(); - const std::vector& in_zp = - dq8ca ? tc.inputs()[2].get_int8_data() : std::vector(); - const std::vector& w4 = - four ? w.get_uint8_data() : std::vector(); // [N, K/2] nibbles - const std::vector& w8 = - four ? std::vector() : w.get_int8_data(); // [N, K] - - auto& ref = out.get_ref_float_data(); - ref.resize(M * N); - for (int64_t m = 0; m < M; ++m) { - const float s_in = dq8ca ? in_scale[m] : 1.0f; - const int zp = dq8ca ? int(in_zp[m]) : 0; - for (int64_t n = 0; n < N; ++n) { - float acc = 0.0f; - for (int64_t k = 0; k < K; ++k) { - float a = inf[m * K + k]; - if (dq8ca) { - float q = std::round(a / s_in) + float(zp); - q = std::min(std::max(q, -128.0f), 127.0f); - a = q - float(zp); - } - int wv; - if (four) { - const uint8_t byte = w4[n * (K / 2) + k / 2]; - const int nib = (k & 1) ? ((byte >> 4) & 0xF) : (byte & 0xF); - wv = nib - 8; - } else { - wv = w8[n * K + k]; - } - const float w_scale = four ? scf[(k / group) * N + n] : scf[n]; - acc += a * float(wv) * w_scale; - } - float r = dq8ca ? acc * s_in : acc; - if (has_bias) { - r += bf[n]; - } - ref[m * N + n] = r; - } - } -} - -// Llama 3.1 8B linear weight shapes (K,N) at prefill M (multiple of 64 so -// coopmat fires). -static const std::vector> kShapes = { - {4096, 4096}, // q_proj / o_proj - {4096, 1024}, // k_proj / v_proj (GQA) - {4096, 14336}, // gate_proj / up_proj - {14336, 4096}, // down_proj -}; -static const std::vector kOps = { - "linear_q4gsw", - "linear_dq8ca_q4gsw"}; -static constexpr int64_t kM = 1024; -static constexpr int64_t kGroup = 128; - -// Generation order: for each op, for each shape -> {Texture3D, Buffer}. -// Summary pairs results [2i]=tiled, [2i+1]=coopmat. -std::vector generate_cases() { - std::vector cases; - // COOPMAT_BENCH_CORRECTNESS_ONLY=1 skips the (slow) M=1024 perf cases and - // runs just the small correctness matrix below. - const bool correctness_only = - std::getenv("COOPMAT_BENCH_CORRECTNESS_ONLY") != nullptr; - if (!correctness_only) { - for (const auto& op : kOps) { - for (const auto& kn : kShapes) { - LinearConfig cfg{kM, kn.first, kn.second, kGroup, op}; - cases.push_back(make_case(cfg, utils::kTexture3D)); // tiled baseline - cases.push_back( - make_case(cfg, utils::kBuffer)); // coopmat (gate-permitting) - } - } - } - // Correctness: small aligned {64,128,64} cases for ALL FOUR ops; the buffer - // case fires the coopmat shader, validated by bench_reference (the perf - // cases above are skipped by it). POSITIVE well-conditioned data (no fp16 - // cancellation): activations are multiples of 1/16 in [0.5,1.375]; int4 - // nibbles in {9..14} (-> weight +1..+6) / int8 weights in {1..6}. For dq8ca - // the per-row activation scale is forced to 1/16 with zp=0 so the dynamic - // int8 quant round-trip is EXACT in both fp16 and fp32 (quantized values - // 8..22) and the fp32 reference is valid. fp16~=fp32 throughout, so a tight - // tolerance validates shader structure (catches zero-subtile bugs) while - // ignoring benign fp16 noise. Texture3D = tiled, Buffer = coopmat. - // Shapes align to BOTH coopmat geometries (64x64x32 legacy, 128x128x16 - // double-buffered); the second shape dispatches a multi-workgroup grid for - // both, covering the gl_WorkGroupID-derived tile offsets in the store - // address math. - static const std::vector kCorrectnessShapes = { - {64, 128, 64, 64, ""}, - {128, 256, 128, 64, ""}, - {128, 128, 128, 64, ""}, - {256, 256, 256, 64, ""}, - // Discriminators for the tiled-texture cube-shape failure: - {128, 128, 256, 64, ""}, // M == K only - {256, 128, 128, 64, ""}, // K == N only - {64, 128, 256, 64, ""}, // K > M, K < N - {256, 128, 64, 64, ""}}; // K < M, K > N - for (const auto& op : kOps) { - for (const auto& shape : kCorrectnessShapes) { - LinearConfig cfg{shape.M, shape.K, shape.N, shape.group_size, op}; - const bool dq = is_dq8ca(op); - const bool four = is_4bit(op); - for (auto st : {utils::kTexture3D, utils::kBuffer}) { - TestCase t = make_case(cfg, st); - auto& hin = t.inputs()[0].get_half_data(); - for (size_t i = 0; i < hin.size(); ++i) { - hin[i] = float_to_half(0.5f + 0.125f * float(i % 8)); - } - const size_t w_idx = dq ? 3 : 1; - if (four) { - auto& wq = t.inputs()[w_idx].get_uint8_data(); - const uint8_t kPos[6] = {0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE}; - for (size_t i = 0; i < wq.size(); ++i) { - wq[i] = kPos[i % 6]; - } - } else { - auto& wq = t.inputs()[w_idx].get_int8_data(); - for (size_t i = 0; i < wq.size(); ++i) { - wq[i] = int8_t(1 + (i % 6)); - } - } - if (dq) { - auto& hs = t.inputs()[1].get_half_data(); - std::fill(hs.begin(), hs.end(), float_to_half(0.0625f)); - auto& zp = t.inputs()[2].get_int8_data(); - std::fill(zp.begin(), zp.end(), int8_t(0)); - // weights were overwritten above -> recompute the sums - if (four) { - compute_weight_sums_4bit_grouped( - t.inputs()[4], - t.inputs()[w_idx], - cfg.K / cfg.group_size, - cfg.N, - cfg.group_size); - } else { - compute_weight_sums(t.inputs()[4], t.inputs()[w_idx], cfg.N, cfg.K); - } - } - t.set_abs_tolerance(0.5f); - t.set_rel_tolerance(0.05f); - cases.push_back(t); - } - } - } - return cases; -} - -int64_t flop_calc(const TestCase& tc) { - const auto& in = tc.inputs()[0].get_tensor_sizes(); - const auto& out = tc.outputs()[0].get_tensor_sizes(); - const int64_t M = in[0], K = in[1], N = out[1]; - return 2 * M * N * K; // MAC = 2 flops -} - -int main() { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "Coopmat vs Tiled quantized-linear microbench (Llama 3.1 8B shapes, M=" - << kM << ")" << std::endl; - print_separator(); - - auto results = execute_test_cases( - generate_cases, - flop_calc, - "CoopmatLinearBench", - /*warmup=*/3, - /*runs=*/5, - /*reference=*/bench_reference); - - // Summary table: pair tiled (even idx) vs coopmat (odd idx) per (op, shape). - // GFLOP/s computed from avg GPU time and 2*M*N*K flops. - if (results.size() < kOps.size() * kShapes.size() * 2) { - return 0; // correctness-only run: no perf cases to summarize - } - auto gflops = [](float time_us, int64_t M, int64_t K, int64_t N) -> float { - return time_us > 0 ? (2.0f * M * N * K) / (time_us * 1e3f) : 0.0f; - }; - // The result's kernel_name is the test-case name; the dispatched shader - // names are in the per-shader timings (dq8ca cases also run a - // quantize_and_pack shader, so pick the linear_* one). - auto linear_kernel = [](const BenchmarkResult& r) -> std::string { - std::string name = r.get_kernel_name(); - for (const auto& st : r.get_shader_timings()) { - if (st.shader_name.find("linear_") != std::string::npos) { - name = st.shader_name; - } - } - return name; - }; - std::cout - << "\n================ SUMMARY: tiled vs coopmat (GFLOP/s) ================\n"; - std::cout << std::left << std::setw(22) << "op" << std::setw(13) - << "shape(K,N)" << std::right << std::setw(10) << "tiled" - << std::setw(10) << "coopmat" << std::setw(9) << "speedup" - << " coopmat kernel\n"; - size_t idx = 0; - for (const auto& op : kOps) { - for (const auto& kn : kShapes) { - const float t_us = results[idx].get_avg_time_us(); - const float c_us = results[idx + 1].get_avg_time_us(); - const std::string coop_kernel = linear_kernel(results[idx + 1]); - const float tiled = gflops(t_us, kM, kn.first, kn.second); - const float coop = gflops(c_us, kM, kn.first, kn.second); - idx += 2; - // If the "coopmat" (buffer) case did not actually pick a _coopmat shader, - // flag it (e.g. shape not gate-eligible). - const bool fired = coop_kernel.find("coopmat") != std::string::npos; - std::cout << std::left << std::setw(22) << op << std::setw(13) - << ("(" + std::to_string(kn.first) + "," + - std::to_string(kn.second) + ")") - << std::right << std::setw(10) << std::fixed - << std::setprecision(1) << tiled << std::setw(10) << coop - << std::setw(8) << std::setprecision(2) - << (tiled > 0 ? coop / tiled : 0.0f) << "x" - << (fired ? " " : " !") << coop_kernel << "\n"; - } - } - std::cout << "(! = buffer case did NOT dispatch a coopmat shader)\n"; - return 0; -} diff --git a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_fpa_q4gsw_linear.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_fpa_q4gsw_linear.cpp deleted file mode 100644 index ef0249d4267..00000000000 --- a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_fpa_q4gsw_linear.cpp +++ /dev/null @@ -1,548 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -// FPA Q4GSW Linear A/B benchmark binary. -// -// Each generated test case has an `impl_selector` arg routed to the test -// op `test_etvk.test_fpa_q4gsw_linear.{gemm,gemv}` in TestFpaQ4gswLinear.cpp: -// -// GEMM (is_gemv=false): -// 0 -> PROD (et_vk.q4gsw_linear.default; dtype-based -// picker) 1 -> GEMM_W_4X8 (forced non-tin GEMM, nc buffer -// weight) 2 -> GEMM_TIN_W_4X8 (forced tin GEMM, nc buffer -// weight) 3 -> LEGACY (et_vk.linear_q4gsw.default -// legacy shaders) -// -// GEMV (is_gemv=true): -// 0 -> PROD (et_vk.q4gsw_linear.default; -// dtype-based picker) 1 -> GEMV_W_4X8 (forced gemv with -// subgroup broadcast) 2 -> GEMV_W_4X8_NOSG (forced gemv -// without subgroup broadcast) 3 -> LEGACY (et_vk.linear_q4gsw.default -// legacy shaders) 13 -> GEMV_COOP_W_4X8_NC_BUFFER (coop GEMV reusing the -// production -// nc-buffer prepack — same payload as -// W_4X8 GEMM/TIN GEMM/sg-GEMV; -// == g1w64 decomposition) -// 14 -> GEMV_COOP_..._G1W64 (force NUM_GROUPS=1, -// WORKERS_PER_GROUP=64) 15 -> GEMV_COOP_..._G4W16 (force -// NUM_GROUPS=4, WORKERS_PER_GROUP=16) 16 -> GEMV_COOP_..._G8W8 (force -// NUM_GROUPS=8, WORKERS_PER_GROUP=8) -// -// Selectors 14-16 pin the coop nc-buffer GEMV to an explicit reduction -// decomposition regardless of N. The production picker (pick_coop_variant_for_N -// in Q4gswLinear.cpp) only chooses g4w16 / g8w8 at PERF-sized N where the -// reference impl is skipped; these forced selectors give g4w16 / g8w8 numeric -// (ACCU) coverage at small N. Production picker behavior is unchanged. -// -// Selector 3 (LEGACY) is the in-prod q4gsw linear path. It uses a different -// prepack (pack_q4_linear_weight) and shader family -// (linear_q4gsw_tiled_* / linear_q4gsw_coop_*); the framework's per-shader -// timing breakdown will pick those up automatically. - -#include -#include -#include -#include -#include - -using namespace executorch::vulkan::prototyping; - -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 300; - -// Linear configuration struct. -struct LinearConfig { - int64_t M; - int64_t K; - int64_t N; - int64_t group_size; - bool has_bias = false; -}; - -// Convert a ValueSpec's input data (float or half) into a flat -// std::vector for use in the reference implementation. -static std::vector input_to_float_vec(const ValueSpec& spec) { - if (spec.dtype == vkapi::kFloat) { - return spec.get_float_data(); - } - if (spec.dtype == vkapi::kHalf) { - const auto& half_data = spec.get_half_data(); - std::vector out(half_data.size()); - for (size_t i = 0; i < half_data.size(); ++i) { - out[i] = half_to_float(half_data[i]); - } - return out; - } - throw std::invalid_argument( - "Reference implementation supports only float/half input dtypes."); -} - -// Create a single test case for the test_fpa_q4gsw_linear.{gemm,gemv} op. -TestCase create_test_case( - const LinearConfig& config, - vkapi::ScalarType dtype, - utils::StorageType storage, - int32_t impl_selector, - bool is_gemv) { - TestCase test_case; - - const int64_t M = config.M; - const int64_t K = config.K; - const int64_t N = config.N; - const int64_t group_size = config.group_size; - - const bool is_performance = - (M > kRefDimSizeLimit || K > kRefDimSizeLimit || N > kRefDimSizeLimit); - const std::string prefix = is_performance ? "PERF" : "ACCU"; - - const std::string dtype_str = dtype_short(dtype); - const std::string shape_str = shape_bracket({M, K}) + "x[" + - std::to_string(N) + "," + std::to_string(K) + "] g" + - std::to_string(group_size); - const std::string storage_str = repr_str(storage, utils::kWidthPacked); - std::string suffix = std::string("[") + (is_gemv ? "gemv" : "gemm") + " s" + - std::to_string(impl_selector) + "]"; - suffix += config.has_bias ? " bias" : " no_bias"; - const std::string test_name = make_test_label( - prefix, dtype_str, dtype_str, shape_str, storage_str, suffix); - test_case.set_name(test_name); - - const std::string op_name = is_gemv ? "test_etvk.test_fpa_q4gsw_linear.gemv" - : "test_etvk.test_fpa_q4gsw_linear.gemm"; - test_case.set_operator_name(op_name); - - // Input: [M, K] - ValueSpec input( - {M, K}, dtype, storage, utils::kWidthPacked, DataGenType::RANDINT); - - // Weight: [N, K/2] uint8 packed 4-bit - ValueSpec weight( - {N, K / 2}, - vkapi::kByte, - storage, - utils::kWidthPacked, - DataGenType::RANDINT4); - weight.set_constant(true); - weight.set_int4(true); - - // Scales: [K/gs, N] matching input dtype (the custom op prepacks scales - // using the input tensor's dtype). - ValueSpec scales( - {K / group_size, N}, - dtype, - storage, - utils::kWidthPacked, - DataGenType::RANDOM_SCALES); - scales.set_constant(true); - - // Group size - ValueSpec gs_spec(static_cast(group_size)); - - // Bias - ValueSpec bias( - {N}, - dtype, - storage, - utils::kWidthPacked, - config.has_bias ? DataGenType::RANDOM : DataGenType::ZEROS); - bias.set_constant(true); - if (!config.has_bias) { - bias.set_none(true); - } - - // impl_selector as int32 - ValueSpec impl_selector_spec(static_cast(impl_selector)); - - // Output: [M, N] - ValueSpec output( - {M, N}, dtype, storage, utils::kWidthPacked, DataGenType::ZEROS); - - // Tolerance: fp16 outputs use relaxed tolerance to account for f16 - // accumulation / rounding. - float base_tol = 0.05f * (static_cast(K) / 64.0f); - float tol = (dtype == vkapi::kHalf) ? (4.0f * base_tol) : base_tol; - test_case.set_abs_tolerance(tol); - - test_case.add_input_spec(input); - test_case.add_input_spec(weight); - test_case.add_input_spec(scales); - test_case.add_input_spec(gs_spec); - test_case.add_input_spec(bias); - test_case.add_input_spec(impl_selector_spec); - test_case.add_output_spec(output); - - return test_case; -} - -// Reference implementation: simple dequant + fp32 GEMM. Only runs for -// small shapes (gate on kRefDimSizeLimit). -void linear_q4gsw_reference_impl(TestCase& test_case) { - int32_t idx = 0; - const ValueSpec& input_spec = test_case.inputs()[idx++]; - const ValueSpec& weight_spec = test_case.inputs()[idx++]; - const ValueSpec& scales_spec = test_case.inputs()[idx++]; - const ValueSpec& gs_spec = test_case.inputs()[idx++]; - const ValueSpec& bias_spec = test_case.inputs()[idx++]; - // impl_selector is not used in the reference impl - ++idx; - - ValueSpec& output_spec = test_case.outputs()[0]; - - auto input_sizes = input_spec.get_tensor_sizes(); - auto output_sizes = output_spec.get_tensor_sizes(); - - int64_t M = input_sizes[0]; - int64_t K = input_sizes[1]; - int64_t N = output_sizes[1]; - int64_t group_size = gs_spec.get_int_value(); - - if (M > kRefDimSizeLimit || K > kRefDimSizeLimit || N > kRefDimSizeLimit) { - throw std::invalid_argument( - "Dimensions exceed limit for reference implementation."); - } - - std::vector input_data = input_to_float_vec(input_spec); - auto& weight_data = weight_spec.get_uint8_data(); - std::vector scales_data = input_to_float_vec(scales_spec); - std::vector bias_data; - if (!bias_spec.is_none()) { - bias_data = input_to_float_vec(bias_spec); - } - - int64_t num_output_elements = M * N; - auto& ref_data = output_spec.get_ref_float_data(); - ref_data.resize(num_output_elements); - - for (int64_t m = 0; m < M; ++m) { - for (int64_t n = 0; n < N; ++n) { - float sum = 0.0f; - for (int64_t k = 0; k < K; ++k) { - float input_val = input_data[m * K + k]; - - int64_t weight_idx = n * (K / 2) + (k / 2); - uint8_t packed = weight_data[weight_idx]; - int8_t nibble = (k % 2 == 0) - ? static_cast(packed & 0x0F) - 8 - : static_cast((packed >> 4) & 0x0F) - 8; - - int64_t group_idx = k / group_size; - float scale = scales_data[group_idx * N + n]; - - sum += input_val * static_cast(nibble) * scale; - } - if (!bias_spec.is_none()) { - sum += bias_data[n]; - } - ref_data[m * N + n] = sum; - } - } -} - -void reference_impl(TestCase& test_case) { - linear_q4gsw_reference_impl(test_case); -} - -// Custom FLOP calculator: 2 * M * K * N for the linear op itself. -int64_t linear_flop_calculator(const TestCase& test_case) { - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - - int64_t M = input_sizes[0]; - int64_t K = input_sizes[1]; - int64_t N = output_sizes[1]; - return 2 * M * K * N; -} - -// Canonical N/K shapes for LLM hidden-size sweeps. -static const std::vector>& get_nk_shapes() { - static const std::vector> kShapes = { - // (K, N) - {1024, 2048}, - {4096, 4096}, - // {4096, 14336}, // Large-N case can make the full benchmark binary - // unstable. - }; - return kShapes; -} - -// GEMM sweep test cases: M in {32, 128, 256} x N/K shapes x dtype x storage -// x impl_selector in kGemmSelectors. -// -// Selector 3 is the legacy in-prod q4gsw linear path -// (et_vk.linear_q4gsw.default) registered in QuantizedLinear.cpp. -std::vector generate_gemm_test_cases() { - std::vector test_cases; - - const std::vector gemm_Ms = {32, 128, 256}; - const int64_t group_size = 32; - - const std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; - const std::vector storages = { - utils::kBuffer, utils::kTexture3D}; - - // Selectors exercised in the GEMM PERF/ACCU sweep: PROD (0), forced non-tin - // / tin GEMM (1, 2), legacy (3). - const std::vector kGemmSelectors = {0, 1, 2, 3}; - - for (int64_t M : gemm_Ms) { - for (const auto& shape : get_nk_shapes()) { - const int64_t K = shape.first; - const int64_t N = shape.second; - LinearConfig cfg{M, K, N, group_size}; - for (auto dtype : dtypes) { - for (auto storage : storages) { - for (int32_t selector : kGemmSelectors) { - test_cases.push_back(create_test_case( - cfg, dtype, storage, selector, /*is_gemv=*/false)); - } - } - } - } - } - - // Non-aligned-N coverage for the W_4X8 GEMM path. The fp32 GEMM issues a - // 16B ivec4 weight load that spans two consecutive (k4, n4) ivec2 tiles - // along N, so N4 must be even (== N a multiple of 8) at the buffer-stride - // level. The prepack pads the weight buffer's row stride to next-even N4 - // and fills the OOB tiles with bias-zero nibbles; these accuracy cases - // exercise that padding path on shapes with N % 8 != 0. Only the new W_4X8 - // family is tested (selectors 0, 1, 2, 5, 6) — selector 3 (LEGACY) uses a - // different prepack and supports arbitrary N. - const std::vector> kNonAlignedNkShapes = { - // (K, N) — K kept under kRefDimSizeLimit so reference impl runs. - {128, 12}, - {128, 20}, - }; - const std::vector kNonAlignedSelectors = {0, 1, 2}; - for (const auto& shape : kNonAlignedNkShapes) { - const int64_t K = shape.first; - const int64_t N = shape.second; - LinearConfig cfg{32, K, N, group_size}; - for (auto dtype : dtypes) { - for (auto storage : storages) { - for (int32_t selector : kNonAlignedSelectors) { - test_cases.push_back(create_test_case( - cfg, dtype, storage, selector, /*is_gemv=*/false)); - } - } - } - } - - // Small ACCU shape (M=32, K=128, N=128) under kRefDimSizeLimit so the - // reference impl runs. Sanity-checks GEMM correctness during iteration. - { - LinearConfig cfg{32, 128, 128, group_size}; - for (auto dtype : {vkapi::kFloat, vkapi::kHalf}) { - test_cases.push_back(create_test_case( - cfg, - dtype, - utils::kTexture3D, - /*impl_selector=*/0, - /*is_gemv=*/false)); - } - } - - // M-tail ACCU shapes. These exercise final partial GEMM tiles for both the - // fp32 direct-input path (tile height 4) and the fp16 TIN path (tile height - // 8). - for (int64_t M : {31, 33}) { - LinearConfig cfg{M, 128, 128, group_size}; - for (auto dtype : {vkapi::kFloat, vkapi::kHalf}) { - test_cases.push_back(create_test_case( - cfg, - dtype, - utils::kTexture3D, - /*impl_selector=*/0, - /*is_gemv=*/false)); - } - } - - return test_cases; -} - -// GEMV sweep test cases: M = 1 x N/K shapes x dtype x storage x -// impl_selector in {0, 1, 2, 3}. -// -// Selector 3 is the legacy in-prod q4gsw linear path -// (et_vk.linear_q4gsw.default) registered in QuantizedLinear.cpp. -std::vector generate_gemv_test_cases() { - std::vector test_cases; - - const int64_t group_size = 32; - - const std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; - const std::vector storages = { - utils::kBuffer, utils::kTexture3D}; - - // ACCU correctness shapes (under kRefDimSizeLimit=300). Exercise selectors - // PROD (0) and forced nosg (2). Forced sg (1) is intentionally skipped: - // sg requires subgroupSize==64 and produces incorrect results on Mali - // (subgroupSize==16); on those devices the PROD picker correctly routes - // to the nosg variant. N must be a multiple of 128 (= 2 * LWG.x) so the - // GEMV shader has no early-exit threads in any workgroup. - const std::vector> kAccuShapes = { - // (K, N) - {128, 128}, - {256, 256}, - }; - // Selector 13 (nc Buffer, reuses production prepack) included for ACCU - // coverage of the coop nc weight-binding variant. - const std::vector kAccuSelectors = {0, 2, 13}; - for (const auto& shape : kAccuShapes) { - const int64_t K = shape.first; - const int64_t N = shape.second; - LinearConfig cfg{1, K, N, group_size}; - for (auto dtype : dtypes) { - for (auto storage : storages) { - for (int32_t selector : kAccuSelectors) { - test_cases.push_back(create_test_case( - cfg, dtype, storage, selector, /*is_gemv=*/true)); - } - } - } - } - - // Forced coop-reduction-decomposition ACCU coverage (selectors 14/15/16 = - // g1w64 / g4w16 / g8w8). The production picker (pick_coop_variant_for_N) - // only selects g4w16 (10244096) at PERF-sized N, where - // every dim exceeds kRefDimSizeLimit=300 so the reference impl is skipped and - // those reduction decompositions get zero numeric validation. These cases - // pin each decomposition regardless of N at small shapes (all dims <= 300) - // so the reference runs and proves g4w16 / g8w8 compute the SAME result as - // the reference — the M=1 decode path actually shipped for Qwen3 / Llama. - // - // Each WG of variant gN produces N*8 outputs (g1w64 -> 8, g4w16 -> 32, - // g8w8 -> 64), so the N values below tile cleanly into all three: N=128 - // (16 / 4 / 2 WGs) and N=256 (32 / 8 / 4 WGs). The shader also handles a - // ragged final WG, but clean tiles keep the test intent unambiguous. K is - // swept over {64, 128, 256} (all multiples of group_size=32 and <= 300) so - // the K-loop reduction is exercised across short and longer accumulations. - const std::vector> kCoopForcedAccuShapes = { - // (K, N) — all dims <= 300 so linear_q4gsw_reference_impl runs. - {64, 128}, - {128, 256}, - {256, 128}, - }; - // 14 -> g1w64, 15 -> g4w16, 16 -> g8w8. g1w64 included for symmetry. - const std::vector kCoopForcedSelectors = {14, 15, 16}; - for (const auto& shape : kCoopForcedAccuShapes) { - const int64_t K = shape.first; - const int64_t N = shape.second; - LinearConfig cfg{1, K, N, group_size}; - for (auto dtype : dtypes) { - for (auto storage : storages) { - for (int32_t selector : kCoopForcedSelectors) { - test_cases.push_back(create_test_case( - cfg, dtype, storage, selector, /*is_gemv=*/true)); - } - } - } - } - - // GEMV PERF selectors: PROD (0), forced sg (1), forced nosg (2), LEGACY (3), - // nc-Buffer coop (13, reuses production prepack — single-format - // prefill+decode). - const std::vector kGemvPerfSelectors = {0, 1, 2, 3, 13}; - for (const auto& shape : get_nk_shapes()) { - const int64_t K = shape.first; - const int64_t N = shape.second; - LinearConfig cfg{1, K, N, group_size}; - for (auto dtype : dtypes) { - for (auto storage : storages) { - for (int32_t selector : kGemvPerfSelectors) { - test_cases.push_back(create_test_case( - cfg, dtype, storage, selector, /*is_gemv=*/true)); - } - } - } - } - - // LLM-decode-shape PERF cells (M=1 GEMV, group_size=32). Mirrors the actual - // per-layer linear shapes seen during decode profiling on Llama 3.2 1B and - // Qwen3 0.6B; the original Phase 2 corpus (1024/2048/4096 x 2048/4096/11008) - // under-samples these and missed the regression where sg-GEMV (selector 1) - // is 15-22% slower per dispatch than LEGACY coop (selector 3) on Adreno 750. - // - // All N values here are multiples of 128 (= 2 * LWG.x for the GEMV shader), - // so the GEMV shader has no early-exit threads. N=512 is the Llama 3.2 1B - // k_proj/v_proj projection (GQA) and is a multiple of 4 (the prepack - // requirement: prepack_q4_w_4x8_nc_buffer enforces N % 4 == 0). - // - // Default storage is fp16 + Tex3D - that's the actual decode config and the - // shape combo where the regression was observed. We additionally exercise - // K=2048,N=2048 under fp32 + Tex3D and fp32 + Buffer to confirm the - // regression isn't fp16-Tex3D-specific. All four selectors (PROD, sg, nosg, - // LEGACY) are exercised. - const std::vector> kLlmGemvShapes = { - // (K, N) - Llama 3.2 1B - {2048, 512}, // k_proj / v_proj (GQA) - {2048, 2048}, // q_proj - {2048, 8192}, // gate_proj / up_proj - {8192, 2048}, // down_proj - // (K, N) - Qwen3 0.6B - {1024, 1024}, // k_proj / v_proj - {1024, 2048}, // q_proj (also overlaps with original corpus) - {1024, 3072}, // gate_proj / up_proj - {3072, 1024}, // down_proj - }; - for (const auto& shape : kLlmGemvShapes) { - const int64_t K = shape.first; - const int64_t N = shape.second; - LinearConfig cfg{1, K, N, group_size}; - for (int32_t selector : kGemvPerfSelectors) { - test_cases.push_back(create_test_case( - cfg, vkapi::kHalf, utils::kTexture3D, selector, /*is_gemv=*/true)); - } - } - // Diversity sanity check: K=2048,N=2048 under fp32 + {Tex3D, Buffer} to - // confirm the regression isn't fp16-Tex3D-specific. - { - LinearConfig cfg{1, 2048, 2048, group_size}; - for (auto storage : {utils::kTexture3D, utils::kBuffer}) { - for (int32_t selector : kGemvPerfSelectors) { - test_cases.push_back(create_test_case( - cfg, vkapi::kFloat, storage, selector, /*is_gemv=*/true)); - } - } - } - - return test_cases; -} - -std::vector generate_all_test_cases() { - auto gemv = generate_gemv_test_cases(); - auto gemm = generate_gemm_test_cases(); - gemv.insert(gemv.end(), gemm.begin(), gemm.end()); - return gemv; -} - -int main(int argc, char* argv[]) { - (void)argc; - (void)argv; - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "FPA Q4GSW Linear A/B Variant Prototyping Framework (gemm + gemv)" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_all_test_cases, - linear_flop_calculator, - "FpaQ4gswLinear", - 3, - 10, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/coopmat_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/coopmat_sets.cpp new file mode 100644 index 00000000000..826bdde55c5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/coopmat_sets.cpp @@ -0,0 +1,154 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q4gsw_linear { + +namespace { + +// POSITIVE well-conditioned data (no fp16 cancellation): activations are +// multiples of 1/16 in [0.5,1.375]; int4 nibbles in {9..14} (-> weight +1..+6) +// / int8 weights in {1..6}. For dq8ca the per-row activation scale is forced to +// 1/16 with zp=0 so the dynamic int8 quant round-trip is EXACT in both fp16 and +// fp32 (quantized values 8..22) and the fp32 reference is valid. fp16~=fp32 +// throughout, so a tight tolerance validates shader structure (catches +// zero-subtile bugs) while ignoring benign fp16 noise. +void set_well_conditioned_data(TestCase& t, const CoopmatLinearConfig& cfg) { + const bool dq = is_dq8ca(cfg.op_name); + const bool four = is_4bit(cfg.op_name); + auto& hin = t.inputs()[0].get_half_data(); + for (size_t i = 0; i < hin.size(); ++i) { + hin[i] = float_to_half(0.5f + 0.125f * float(i % 8)); + } + const size_t w_idx = dq ? 3 : 1; + if (four) { + auto& wq = t.inputs()[w_idx].get_uint8_data(); + const std::array kPos = {0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE}; + for (size_t i = 0; i < wq.size(); ++i) { + wq[i] = kPos[i % 6]; + } + } else { + auto& wq = t.inputs()[w_idx].get_int8_data(); + for (size_t i = 0; i < wq.size(); ++i) { + wq[i] = int8_t(1 + (i % 6)); + } + } + if (dq) { + auto& hs = t.inputs()[1].get_half_data(); + std::fill(hs.begin(), hs.end(), float_to_half(0.0625f)); + auto& zp = t.inputs()[2].get_int8_data(); + std::fill(zp.begin(), zp.end(), int8_t(0)); + // weights were overwritten above -> recompute the sums + if (four) { + compute_weight_sums_4bit_grouped( + t.inputs()[4], + t.inputs()[w_idx], + cfg.K / cfg.group_size, + cfg.N, + cfg.group_size); + } else { + compute_weight_sums(t.inputs()[4], t.inputs()[w_idx], cfg.N, cfg.K); + } + } + t.set_abs_tolerance(0.5f); + t.set_rel_tolerance(0.05f); +} + +std::vector generate_coopmat_test_cases( + const std::vector& shapes, + const std::vector& ops, + const std::vector& storage_types, + bool well_conditioned_data) { + std::vector cases; + for (const auto& op : ops) { + for (const auto& shape : shapes) { + CoopmatLinearConfig cfg{shape.M, shape.K, shape.N, shape.group_size, op}; + for (auto st : storage_types) { + TestCase t = make_case(cfg, st); + if (well_conditioned_data) { + set_well_conditioned_data(t, cfg); + } + cases.push_back(std::move(t)); + } + } + } + return cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q4gsw_linear", "coopmat_correctness") { + // Correctness: small aligned {64,128,64} cases for ALL FOUR ops; the buffer + // case fires the coopmat shader, validated by bench_reference (the + // performance cases are skipped by it). Texture3D = tiled, Buffer = coopmat. + // Shapes align to BOTH coopmat geometries (64x64x32 legacy, 128x128x16 + // double-buffered); the second shape dispatches a multi-workgroup grid for + // both, covering the gl_WorkGroupID-derived tile offsets in the store + // address math. + return { + generate_coopmat_test_cases( + { + {64, 128, 64, 64, ""}, + {128, 256, 128, 64, ""}, + {128, 128, 128, 64, ""}, + {256, 256, 256, 64, ""}, + // Discriminators for the tiled-texture cube-shape failure: + {128, 128, 256, 64, ""}, // M == K only + {256, 128, 128, 64, ""}, // K == N only + {64, 128, 256, 64, ""}, // K > M, K < N + {256, 128, 64, 64, ""}, // K < M, K > N + }, + /*ops=*/{"linear_q4gsw", "linear_dq8ca_q4gsw"}, + /*storage_types=*/{utils::kTexture3D, utils::kBuffer}, + /*well_conditioned_data=*/true), + bench_reference, + flop_calc, + /*warmup_runs=*/3, + /*benchmark_runs=*/5}; +} + +namespace { + +constexpr int64_t kM = 1024; +constexpr int64_t kGroup = 128; + +} // namespace + +REGISTER_TEST_CASE_SET("q4gsw_linear", "coopmat_performance") { + return { + generate_coopmat_test_cases( + { + // Llama 3.1 8B linear weight shapes (K,N) at prefill M (multiple + // of 64 so coopmat fires). + {kM, 4096, 4096, kGroup, ""}, // q_proj / o_proj + {kM, 4096, 1024, kGroup, ""}, // k_proj / v_proj (GQA) + {kM, 4096, 14336, kGroup, ""}, // gate_proj / up_proj + {kM, 14336, 4096, kGroup, ""}, // down_proj + }, + /*ops=*/{"linear_q4gsw", "linear_dq8ca_q4gsw"}, + // Texture3D selects the tiled shader, Buffer the coopmat shader. + /*storage_types=*/{utils::kTexture3D, utils::kBuffer}, + /*well_conditioned_data=*/false), + bench_reference, + flop_calc, + /*warmup_runs=*/3, + /*benchmark_runs=*/5}; +} + +} // namespace q4gsw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/debug_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/debug_set.cpp new file mode 100644 index 00000000000..64e1d6999c7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/debug_set.cpp @@ -0,0 +1,58 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q4gsw_linear { + +namespace { + +std::vector generate_debug_test_cases( + const LinearConfig& config, + const std::vector& storage_types, + const std::vector& input_dtypes) { + std::vector test_cases; + + for (const auto& storage_type : storage_types) { + for (const auto& input_dtype : input_dtypes) { + test_cases.push_back( + create_test_case_from_config(config, storage_type, input_dtype)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q4gsw_linear", "debug") { + // Single simple configuration for debugging + return { + generate_debug_test_cases( + { + 8, // Batch size + 16, // Input features + 16, // Output features + 8, // Group size + true, // has_bias + "simple", // test_case_name + "linear_dq8ca_q4gsw", // op_name + }, + /*storage_types=*/{utils::kTexture3D, utils::kBuffer}, + /*input_dtypes=*/{vkapi::kFloat}), + reference_impl, + quantized_linear_flop_calculator}; +} + +} // namespace q4gsw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/fpa_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/fpa_sets.cpp new file mode 100644 index 00000000000..9e7902a6008 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/fpa_sets.cpp @@ -0,0 +1,264 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q4gsw_linear { + +namespace { + +// One case per M x (K, N) x dtype x storage x impl_selector, in that nesting +// order. +struct FpaSweep { + std::vector Ms; + std::vector> nk_shapes; // (K, N) + std::vector dtypes; + std::vector storages; + std::vector impl_selectors; + bool is_gemv; +}; + +std::vector generate_fpa_test_cases( + int64_t group_size, + const std::vector& sweeps) { + std::vector test_cases; + for (const auto& sweep : sweeps) { + for (int64_t M : sweep.Ms) { + for (const auto& shape : sweep.nk_shapes) { + const int64_t K = shape.first; + const int64_t N = shape.second; + FpaLinearConfig cfg{M, K, N, group_size}; + for (auto dtype : sweep.dtypes) { + for (auto storage : sweep.storages) { + for (int32_t selector : sweep.impl_selectors) { + test_cases.push_back(create_test_case( + cfg, dtype, storage, selector, sweep.is_gemv)); + } + } + } + } + } + } + return test_cases; +} + +const std::vector kDtypes = {vkapi::kFloat, vkapi::kHalf}; +const std::vector kStorages = { + utils::kBuffer, + utils::kTexture3D}; + +// ACCU correctness shapes (under kRefDimSizeLimit=300). Exercise selectors +// PROD (0) and forced nosg (2). Forced sg (1) is intentionally skipped: +// sg requires subgroupSize==64 and produces incorrect results on Mali +// (subgroupSize==16); on those devices the PROD picker correctly routes +// to the nosg variant. N must be a multiple of 128 (= 2 * LWG.x) so the +// GEMV shader has no early-exit threads in any workgroup. +const std::vector> kAccuShapes = { + // (K, N) + {128, 128}, + {256, 256}, +}; +// Selector 13 (nc Buffer, reuses production prepack) included for ACCU +// coverage of the coop nc weight-binding variant. +const std::vector kAccuSelectors = {0, 2, 13}; + +// Forced coop-reduction-decomposition ACCU coverage (selectors 14/15/16 = +// g1w64 / g4w16 / g8w8). The production picker (pick_coop_variant_for_N) +// only selects g4w16 (10244096) at PERF-sized N, where +// every dim exceeds kRefDimSizeLimit=300 so the reference impl is skipped and +// those reduction decompositions get zero numeric validation. These cases +// pin each decomposition regardless of N at small shapes (all dims <= 300) +// so the reference runs and proves g4w16 / g8w8 compute the SAME result as +// the reference — the M=1 decode path actually shipped for Qwen3 / Llama. +// +// Each WG of variant gN produces N*8 outputs (g1w64 -> 8, g4w16 -> 32, +// g8w8 -> 64), so the N values below tile cleanly into all three: N=128 +// (16 / 4 / 2 WGs) and N=256 (32 / 8 / 4 WGs). The shader also handles a +// ragged final WG, but clean tiles keep the test intent unambiguous. K is +// swept over {64, 128, 256} (all multiples of group_size=32 and <= 300) so +// the K-loop reduction is exercised across short and longer accumulations. +const std::vector> kCoopForcedAccuShapes = { + // (K, N) — all dims <= 300 so linear_q4gsw_reference_impl runs. + {64, 128}, + {128, 256}, + {256, 128}, +}; +// 14 -> g1w64, 15 -> g4w16, 16 -> g8w8. g1w64 included for symmetry. +const std::vector kCoopForcedSelectors = {14, 15, 16}; + +// Non-aligned-N coverage for the W_4X8 GEMM path. The fp32 GEMM issues a +// 16B ivec4 weight load that spans two consecutive (k4, n4) ivec2 tiles +// along N, so N4 must be even (== N a multiple of 8) at the buffer-stride +// level. The prepack pads the weight buffer's row stride to next-even N4 +// and fills the OOB tiles with bias-zero nibbles; these accuracy cases +// exercise that padding path on shapes with N % 8 != 0. Only the new W_4X8 +// family is tested (selectors 0, 1, 2, 5, 6) — selector 3 (LEGACY) uses a +// different prepack and supports arbitrary N. +const std::vector> kNonAlignedNkShapes = { + // (K, N) — K kept under kRefDimSizeLimit so reference impl runs. + {128, 12}, + {128, 20}, +}; +const std::vector kNonAlignedSelectors = {0, 1, 2}; + +} // namespace + +REGISTER_TEST_CASE_SET("q4gsw_linear", "fpa_correctness") { + return { + generate_fpa_test_cases( + /*group_size=*/32, + { + // GEMV + {{1}, + kAccuShapes, + kDtypes, + kStorages, + kAccuSelectors, + /*is_gemv=*/true}, + {{1}, + kCoopForcedAccuShapes, + kDtypes, + kStorages, + kCoopForcedSelectors, + /*is_gemv=*/true}, + // GEMM + {{32}, + kNonAlignedNkShapes, + kDtypes, + kStorages, + kNonAlignedSelectors, + /*is_gemv=*/false}, + // Small ACCU shape (M=32, K=128, N=128) under kRefDimSizeLimit so + // the reference impl runs. Sanity-checks GEMM correctness during + // iteration. + {{32}, + {{128, 128}}, + kDtypes, + {utils::kTexture3D}, + /*impl_selectors=*/{0}, + /*is_gemv=*/false}, + // M-tail ACCU shapes. These exercise final partial GEMM tiles for + // both the fp32 direct-input path (tile height 4) and the fp16 + // TIN path (tile height 8). + {{31, 33}, + {{128, 128}}, + kDtypes, + {utils::kTexture3D}, + /*impl_selectors=*/{0}, + /*is_gemv=*/false}, + }), + fpa_reference_impl, + linear_flop_calculator, + /*warmup_runs=*/3, + /*benchmark_runs=*/10}; +} + +namespace { + +// Canonical N/K shapes for LLM hidden-size sweeps. +const std::vector> kNkShapes = { + // (K, N) + {1024, 2048}, + {4096, 4096}, + // {4096, 14336}, // Large-N case can make the full benchmark binary + // unstable. +}; + +// GEMV PERF selectors: PROD (0), forced sg (1), forced nosg (2), LEGACY (3), +// nc-Buffer coop (13, reuses production prepack — single-format +// prefill+decode). +const std::vector kGemvPerfSelectors = {0, 1, 2, 3, 13}; + +// LLM-decode-shape PERF cells (M=1 GEMV, group_size=32). Mirrors the actual +// per-layer linear shapes seen during decode profiling on Llama 3.2 1B and +// Qwen3 0.6B; the original Phase 2 corpus (1024/2048/4096 x 2048/4096/11008) +// under-samples these and missed the regression where sg-GEMV (selector 1) +// is 15-22% slower per dispatch than LEGACY coop (selector 3) on Adreno 750. +// +// All N values here are multiples of 128 (= 2 * LWG.x for the GEMV shader), +// so the GEMV shader has no early-exit threads. N=512 is the Llama 3.2 1B +// k_proj/v_proj projection (GQA) and is a multiple of 4 (the prepack +// requirement: prepack_q4_w_4x8_nc_buffer enforces N % 4 == 0). +// +// Default storage is fp16 + Tex3D - that's the actual decode config and the +// shape combo where the regression was observed. We additionally exercise +// K=2048,N=2048 under fp32 + Tex3D and fp32 + Buffer to confirm the +// regression isn't fp16-Tex3D-specific. All four selectors (PROD, sg, nosg, +// LEGACY) are exercised. +const std::vector> kLlmGemvShapes = { + // (K, N) - Llama 3.2 1B + {2048, 512}, // k_proj / v_proj (GQA) + {2048, 2048}, // q_proj + {2048, 8192}, // gate_proj / up_proj + {8192, 2048}, // down_proj + // (K, N) - Qwen3 0.6B + {1024, 1024}, // k_proj / v_proj + {1024, 2048}, // q_proj (also overlaps with original corpus) + {1024, 3072}, // gate_proj / up_proj + {3072, 1024}, // down_proj +}; + +// Selectors exercised in the GEMM PERF/ACCU sweep: PROD (0), forced non-tin +// / tin GEMM (1, 2), legacy (3). +// +// Selector 3 is the legacy in-prod q4gsw linear path +// (et_vk.linear_q4gsw.default) registered in QuantizedLinear.cpp. +const std::vector kGemmSelectors = {0, 1, 2, 3}; + +} // namespace + +REGISTER_TEST_CASE_SET("q4gsw_linear", "fpa_performance") { + return { + generate_fpa_test_cases( + /*group_size=*/32, + { + // GEMV sweep: M = 1 x N/K shapes x dtype x storage x + // impl_selector. + {{1}, + kNkShapes, + kDtypes, + kStorages, + kGemvPerfSelectors, + /*is_gemv=*/true}, + {{1}, + kLlmGemvShapes, + {vkapi::kHalf}, + {utils::kTexture3D}, + kGemvPerfSelectors, + /*is_gemv=*/true}, + // Diversity sanity check: K=2048,N=2048 under fp32 + {Tex3D, + // Buffer} to confirm the regression isn't fp16-Tex3D-specific. + {{1}, + {{2048, 2048}}, + {vkapi::kFloat}, + {utils::kTexture3D, utils::kBuffer}, + kGemvPerfSelectors, + /*is_gemv=*/true}, + // GEMM sweep: M in {32, 128, 256} x N/K shapes x dtype x storage + // x impl_selector. + {{32, 128, 256}, + kNkShapes, + kDtypes, + kStorages, + kGemmSelectors, + /*is_gemv=*/false}, + }), + fpa_reference_impl, + linear_flop_calculator, + /*warmup_runs=*/3, + /*benchmark_runs=*/10}; +} + +} // namespace q4gsw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/q4gsw_linear_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/q4gsw_linear_sets.cpp new file mode 100644 index 00000000000..be0e57dbe42 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/q4gsw_linear_sets.cpp @@ -0,0 +1,164 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q4gsw_linear { + +namespace { + +std::vector generate_test_cases( + const std::vector& configs, + const std::vector& storage_types, + const std::vector& input_dtypes, + const std::vector& ops, + bool include_narrow_tile_cases = false) { + std::vector test_cases; + + const bool supports_int8_dot_product = + vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product(); + + for (auto config : configs) { + std::string prefix = + (config.M < kRefDimSizeLimit && config.K < kRefDimSizeLimit && + config.N < kRefDimSizeLimit) + ? "correctness_" + : "performance_"; + std::string generated_test_case_name = prefix + std::to_string(config.M) + + "_" + std::to_string(config.K) + "_" + std::to_string(config.N) + "_g" + + std::to_string(config.group_size); + if (!config.has_bias) { + generated_test_case_name += "_no_bias"; + } + + config.test_case_name = generated_test_case_name; + + for (const auto& storage_type : storage_types) { + for (const auto& input_dtype : input_dtypes) { + for (const auto& op : ops) { + if (is_dq8ca(op) && !supports_int8_dot_product) { + continue; + } + LinearConfig op_config = config; + op_config.op_name = op; + test_cases.push_back(create_test_case_from_config( + op_config, storage_type, input_dtype)); + } + } + } + } + + if (include_narrow_tile_cases) { + for (int64_t M : {2, 3, 5}) { + for (bool has_bias : {true, false}) { + LinearConfig config; + config.M = M; + config.K = 64; + config.N = 32; + config.group_size = 32; + config.has_bias = has_bias; + config.op_name = "linear_dq8ca_q4gsw"; + config.test_case_name = "correctness_tiledm2_M" + std::to_string(M) + + (has_bias ? "" : "_no_bias"); + + for (const auto& storage_type : storage_types) { + for (const auto& input_dtype : input_dtypes) { + TestCase test_case = + create_test_case_from_config(config, storage_type, input_dtype); + test_case.set_name(test_case.name() + " [tiledm2]"); + test_case.set_force_narrow_int4_tile(true); + test_cases.push_back(test_case); + } + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q4gsw_linear", "correctness") { + return { + generate_test_cases( + { + // // Gemv test cases + // {1, 128, 64, 32}, + // {1, 256, 128, 64}, + // Gemm + {4, 64, 32, 16}, + {4, 128, 64, 32}, + {4, 256, 128, 64}, + {32, 64, 32, 16}, + {32, 128, 64, 32}, + {32, 256, 128, 64}, + {256, 128, 128, 32}, // M=256 > K=128; all dims < kRefDimSizeLimit + + // Coopmat-eligible correctness shapes (M%64==0, N%64==0, + // K%32==0, group_size%32==0). The Buffer+Half variant fires + // linear_q4gsw_coopmat / linear_dq8ca_q4gsw_coopmat and is + // validated against the CPU reference. + {64, 64, 64, 64}, + {64, 128, 64, 64}, + {64, 256, 128, 128}, + // With bias + {4, 64, 32, 16, true}, + {4, 128, 64, 32, true}, + {32, 128, 64, 32, true}, + // NOTE: coopmat correctness coverage is NOT in this list. The + // coopmat dispatch gate requires M%64==0, N%64==0, K%32==0; the + // smallest qualifying shape (M=64, K=64, N=64) produces enough + // cancellation outputs that fp16 accumulation drift exceeds any + // reasonable tolerance against the fp32 reference. Validating + // the coopmat shader needs a different strategy (e.g. + // positive-only inputs, or simulating fp16 accumulation in the + // reference). + }, + /*storage_types=*/{utils::kTexture3D, utils::kBuffer}, + // Both fp32 and fp16 activations, so the test covers the _float and + // _half SPIR-V variants of each linear shader. Llama-on-Vulkan + // exports run with backend.vulkan.force_fp16=True, so the _half + // variants are the ones we actually hit in production. + /*input_dtypes=*/{vkapi::kFloat, vkapi::kHalf}, + // Activation+weight quantized and weight-only quantized. + /*ops=*/{"linear_dq8ca_q4gsw", "linear_q4gsw"}, + /*include_narrow_tile_cases=*/true), + reference_impl, + quantized_linear_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q4gsw_linear", "performance") { + return { + generate_test_cases( + { + // A couple of representative performance shapes + // (coopmat-eligible, M % 64 == 0). The full Llama 3.1 8B prefill + // sweep lived here during the study; trimmed to keep this a fast + // unit test. + {128, 2048, 2048, 128}, + {1024, 4096, 4096, 128}, + }, + /*storage_types=*/{utils::kTexture3D, utils::kBuffer}, + // fp16 is the production path (backend.vulkan.force_fp16=True); fp32 + // covers the _float shader variants. + /*input_dtypes=*/{vkapi::kFloat, vkapi::kHalf}, + // Activation+weight quantized and weight-only quantized. + /*ops=*/{"linear_dq8ca_q4gsw", "linear_q4gsw"}), + reference_impl, + quantized_linear_flop_calculator}; +} + +} // namespace q4gsw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/test_q8csw_conv2d.cpp b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/q8csw_conv2d.cpp similarity index 74% rename from backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/test_q8csw_conv2d.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/q8csw_conv2d.cpp index 558f9228355..8af65739d8e 100644 --- a/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/test_q8csw_conv2d.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/q8csw_conv2d.cpp @@ -4,20 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include -#include -#include -#include -#include - -#include - -using namespace executorch::vulkan::prototyping; +#include -using namespace vkcompute; +#include +#include +#include +#include +#include -static constexpr int64_t kRefDimSizeLimit = 100; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_conv2d { // Utility function to create a test case from a Conv2dConfig TestCase create_test_case_from_config( @@ -178,196 +176,40 @@ TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for quantized conv2d operation (for debugging) -std::vector generate_quantized_conv2d_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - Conv2dConfig config = { - OutInChannels(8, 3), // channels (out, in) - InputSize2D(8, 8), // input_size (h, w) - KernelSize(3, 3), // kernel - Stride(1, 1), // stride - Padding(0, 0), // padding - Dilation(1, 1), // dilation - 1, // groups - }; - - // Test with both storage types and data types for completeness - std::vector storage_types = {utils::kTexture3D}; - std::vector float_types = {vkapi::kFloat}; - - // Generate test cases for each combination - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, input_dtype)); - } +// Custom FLOP calculator for quantized conv2d operation +int64_t quantized_conv2d_flop_calculator(const TestCase& test_case) { + int kernel_idx = 4; + if (test_case.operator_name().find("q8ta") != std::string::npos) { + kernel_idx = 7; } + // Get input and weight dimensions + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - return test_cases; -} + const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); -// Generate test cases for quantized conv2d operation -std::vector generate_quantized_conv2d_test_cases() { - std::vector test_cases; - - std::vector configs = { - {OutInChannels(32, 3), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(32, 16), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - // One output channel case - {OutInChannels(1, 32), - InputSize2D(55, 55), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - - // Stride 2 convolutions - {OutInChannels(32, 3), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - // Different kernel sizes - {OutInChannels(32, 16), - InputSize2D(28, 28), - KernelSize(5, 5), - Stride(1, 1), - Padding(2, 2), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(14, 14), - KernelSize(7, 7), - Stride(1, 1), - Padding(3, 3), - Dilation(1, 1), - 1}, - - // Dilated convolutions - {OutInChannels(32, 16), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(1, 1), - Padding(2, 2), - Dilation(2, 2), - 1}, - {OutInChannels(64, 32), - InputSize2D(16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(3, 3), - Dilation(3, 3), - 1}, - - // Grouped convolutions - {OutInChannels(32, 32), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 4}, - {OutInChannels(64, 64), - InputSize2D(16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 8}, - // Performance test cases - {OutInChannels(256, 128), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(128, 64), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(128, 1024), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}}; - - // Test with different storage types and data types - std::vector storage_types = {utils::kTexture3D}; - - // Generate test cases for each combination - for (auto& config : configs) { - for (const auto& storage_type : storage_types) { - // Generate test case name programmatically - bool is_performance = config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit || - config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - std::string prefix = is_performance ? "performance_" : "correctness_"; - std::string suffix = std::to_string(config.channels.out) + "/" + - std::to_string(config.channels.in) + "_" + - std::to_string(config.input_size.h) + "/" + - std::to_string(config.input_size.w) + "_" + - std::to_string(config.kernel.h) + "/" + - std::to_string(config.kernel.w); - - config.op_name = "conv2d_q8ta_q8csw"; - config.test_case_name = prefix + suffix; - // The default operator tested is activation + weight quantized conv2d; - // however, only test this if the int8 dot product extension is supported - if (vkcompute::api::context() - ->adapter_ptr() - ->supports_int8_dot_product()) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, vkapi::kFloat)); - } + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t C_out = output_sizes[1]; + int64_t K_h = kernel_sizes[0]; + int64_t K_w = kernel_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; - Conv2dConfig wo_quant_config = config; - wo_quant_config.op_name = "conv2d_q8csw"; - test_cases.push_back(create_test_case_from_config( - wo_quant_config, storage_type, vkapi::kFloat)); - } - } + // Calculate FLOPs for quantized conv2d operation + // Each output element requires: + // - C_in * K_h * K_w multiply-accumulate operations + // - Additional operations for quantization/dequantization + int64_t output_elements = N * C_out * H_out * W_out; + int64_t ops_per_output = C_in * K_h * K_w; + + int64_t flop = output_elements * (ops_per_output); - return test_cases; + return flop; } +namespace { + // Reference implementation for weight only quantized conv2d (fp accumulation) void conv2d_q8csw_reference_impl(TestCase& test_case) { int32_t idx = 0; @@ -444,7 +286,8 @@ void conv2d_q8csw_reference_impl(TestCase& test_case) { auto& ref_data = output_spec.get_ref_float_data(); ref_data.resize(num_output_elements); - const int in_features = utils::align_up_4(C_in_per_group * K_h * K_w); + const int in_features = + utils::safe_downcast(utils::align_up_4(C_in_per_group * K_h * K_w)); // Perform weight-only quantized conv2d operation (fp accumulation) for (int64_t n = 0; n < N; ++n) { @@ -584,7 +427,8 @@ void conv2d_q8ta_q8csw_reference_impl(TestCase& test_case) { auto& ref_data = output_spec.get_ref_float_data(); ref_data.resize(num_output_elements); - const int in_features = utils::align_up_4(C_in_per_group * K_h * K_w); + const int in_features = + utils::safe_downcast(utils::align_up_4(C_in_per_group * K_h * K_w)); // Perform activation and weight quantized conv2d operation (int accumulation) for (int64_t n = 0; n < N; ++n) { @@ -674,6 +518,8 @@ void conv2d_q8ta_q8csw_reference_impl(TestCase& test_case) { } } +} // namespace + void reference_impl(TestCase& test_case) { if (test_case.operator_name().find("q8ta") != std::string::npos) { conv2d_q8ta_q8csw_reference_impl(test_case); @@ -682,58 +528,7 @@ void reference_impl(TestCase& test_case) { } } -// Custom FLOP calculator for quantized conv2d operation -int64_t quantized_conv2d_flop_calculator(const TestCase& test_case) { - int kernel_idx = 4; - if (test_case.operator_name().find("q8ta") != std::string::npos) { - kernel_idx = 7; - } - // Get input and weight dimensions - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - - const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t C_out = output_sizes[1]; - int64_t K_h = kernel_sizes[0]; - int64_t K_w = kernel_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - // Calculate FLOPs for quantized conv2d operation - // Each output element requires: - // - C_in * K_h * K_w multiply-accumulate operations - // - Additional operations for quantization/dequantization - int64_t output_elements = N * C_out * H_out * W_out; - int64_t ops_per_output = C_in * K_h * K_w; - - int64_t flop = output_elements * (ops_per_output); - - return flop; -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(true); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Quantized Conv2d Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - // Execute test cases using the new framework with custom FLOP calculator - auto results = execute_test_cases( - generate_quantized_conv2d_test_cases, - quantized_conv2d_flop_calculator, - "QuantizedConv2d", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8csw_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/q8csw_conv2d.h b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/q8csw_conv2d.h new file mode 100644 index 00000000000..0cf3f5a4278 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/q8csw_conv2d.h @@ -0,0 +1,32 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_conv2d { + +constexpr int64_t kRefDimSizeLimit = 100; + +TestCase create_test_case_from_config( + const Conv2dConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype); + +void reference_impl(TestCase& test_case); + +int64_t quantized_conv2d_flop_calculator(const TestCase& test_case); + +} // namespace q8csw_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/test_sets/q8csw_conv2d_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/test_sets/q8csw_conv2d_sets.cpp new file mode 100644 index 00000000000..6bee244b04d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/test_sets/q8csw_conv2d_sets.cpp @@ -0,0 +1,202 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_conv2d { + +namespace { + +std::vector generate_test_cases( + const std::vector& configs, + const std::vector& storage_types) { + std::vector test_cases; + + // Generate test cases for each combination + for (auto config : configs) { + for (const auto& storage_type : storage_types) { + // Generate test case name programmatically + bool is_performance = config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit || + config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + std::string prefix = is_performance ? "performance_" : "correctness_"; + std::string suffix = std::to_string(config.channels.out) + "/" + + std::to_string(config.channels.in) + "_" + + std::to_string(config.input_size.h) + "/" + + std::to_string(config.input_size.w) + "_" + + std::to_string(config.kernel.h) + "/" + + std::to_string(config.kernel.w); + + config.op_name = "conv2d_q8ta_q8csw"; + config.test_case_name = prefix + suffix; + // The default operator tested is activation + weight quantized conv2d; + // however, only test this if the int8 dot product extension is supported + if (vkcompute::api::context() + ->adapter_ptr() + ->supports_int8_dot_product()) { + test_cases.push_back( + create_test_case_from_config(config, storage_type, vkapi::kFloat)); + } + + Conv2dConfig wo_quant_config = config; + wo_quant_config.op_name = "conv2d_q8csw"; + test_cases.push_back(create_test_case_from_config( + wo_quant_config, storage_type, vkapi::kFloat)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8csw_conv2d", "correctness") { + const std::vector configs = { + {OutInChannels(32, 3), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(32, 16), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + // One output channel case + {OutInChannels(1, 32), + InputSize2D(55, 55), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + + // Stride 2 convolutions + {OutInChannels(32, 3), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + // Different kernel sizes + {OutInChannels(32, 16), + InputSize2D(28, 28), + KernelSize(5, 5), + Stride(1, 1), + Padding(2, 2), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(14, 14), + KernelSize(7, 7), + Stride(1, 1), + Padding(3, 3), + Dilation(1, 1), + 1}, + + // Dilated convolutions + {OutInChannels(32, 16), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(1, 1), + Padding(2, 2), + Dilation(2, 2), + 1}, + {OutInChannels(64, 32), + InputSize2D(16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(3, 3), + Dilation(3, 3), + 1}, + + // Grouped convolutions + {OutInChannels(32, 32), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 4}, + {OutInChannels(64, 64), + InputSize2D(16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 8}, + }; + const std::vector storage_types = {utils::kTexture3D}; + + return { + generate_test_cases(configs, storage_types), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8csw_conv2d", "performance") { + const std::vector configs = { + // Performance test cases + {OutInChannels(256, 128), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(128, 64), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(128, 1024), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + }; + const std::vector storage_types = {utils::kTexture3D}; + + return { + generate_test_cases(configs, storage_types), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8csw_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_q8csw_linear.cpp b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/q8csw_linear.cpp similarity index 75% rename from backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_q8csw_linear.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8csw_linear/q8csw_linear.cpp index 33c6947e167..4718ab4310e 100644 --- a/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_q8csw_linear.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/q8csw_linear.cpp @@ -4,29 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include -#include -#include -#include - -#include - -using namespace executorch::vulkan::prototyping; +#include -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 300; +#include +#include +#include +#include +#include -// Linear configuration struct -struct LinearConfig { - int64_t M; // Batch size / number of rows in input - int64_t K; // Input features / columns in input, rows in weight - int64_t N; // Output features / columns in weight - bool has_bias = true; - std::string test_case_name = "placeholder"; - std::string op_name = "linear_q8ta_q8csw"; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_linear { // Utility function to create a test case from a LinearConfig TestCase create_test_case_from_config( @@ -158,95 +147,42 @@ TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for quantized linear operation (for debugging) -std::vector generate_quantized_linear_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - int M = 4; - int K = 4; - int N = 4; - - LinearConfig config = { - M, // Batch size - K, // Input features - N, // Output features - true, // has_bias - "simple", // test_case_name - }; - - // Test with both storage types and data types for completeness - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - std::vector float_types = {vkapi::kFloat}; - - // Generate test cases for each combination - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, input_dtype)); - } +int64_t quantized_linear_flop_calculator(const TestCase& test_case) { + int input_idx = 0; + int weight_idx = 1; + if (test_case.operator_name().find("q8ta") != std::string::npos) { + input_idx = 0; + weight_idx = 3; } - return test_cases; -} + // Get input and weight dimensions + const auto& input_sizes = test_case.inputs()[input_idx].get_tensor_sizes(); + const auto& weight_sizes = test_case.inputs()[weight_idx].get_tensor_sizes(); -// Generate test cases for quantized linear operation -std::vector generate_quantized_linear_test_cases() { - std::vector test_cases; - - std::vector configs = { - {4, 64, 32}, - {4, 128, 64}, - {4, 256, 128}, - {32, 64, 32}, - {32, 128, 64}, - {32, 256, 128}, - // No bias tests - {32, 128, 64, false}, - {32, 256, 128, false}, - {256, 2048, 2048}, - {512, 2048, 2048}, - {1024, 2048, 2048}, - }; - - // Test with different storage types and data types - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - for (auto config : configs) { - std::string prefix = - (config.M < kRefDimSizeLimit && config.K < kRefDimSizeLimit && - config.N < kRefDimSizeLimit) - ? "correctness_" - : "performance_"; - std::string generated_test_case_name = prefix + std::to_string(config.M) + - "_" + std::to_string(config.K) + "_" + std::to_string(config.N); - if (!config.has_bias) { - generated_test_case_name += "_no_bias"; - } + int64_t batch_size = input_sizes[0]; + int64_t in_features = input_sizes[1]; + int64_t out_features = weight_sizes[0]; - config.test_case_name = generated_test_case_name; + // Calculate FLOPs for quantized linear operation + // Each output element requires: + // - in_features multiply-accumulate operations + // - Additional operations for quantization/dequantization + int64_t output_elements = batch_size * out_features; + int64_t ops_per_output = in_features; - for (const auto& storage_type : storage_types) { - if (vkcompute::api::context() - ->adapter_ptr() - ->supports_int8_dot_product()) { - // Test both activation+weight quantized and weight only quantized - test_cases.push_back( - create_test_case_from_config(config, storage_type, vkapi::kFloat)); - } + // Add quantization overhead (approximate) + // - Dequantize input: 1 op per input element used + // - Dequantize weight: 1 op per weight element used + // - Add bias: 1 op per output element + int64_t quantization_ops = ops_per_output + 1; // Simplified estimate - LinearConfig wo_quant_config = config; - wo_quant_config.op_name = "linear_q8csw"; - test_cases.push_back(create_test_case_from_config( - wo_quant_config, storage_type, vkapi::kFloat)); - } - } + int64_t flop = output_elements * (ops_per_output + quantization_ops); - return test_cases; + return flop; } +namespace { + // Reference implementation for weight only quantized linear void linear_q8csw_reference_impl(TestCase& test_case) { int32_t idx = 0; @@ -425,6 +361,8 @@ void linear_q8ta_q8csw_reference_impl(TestCase& test_case) { } } +} // namespace + void reference_impl(TestCase& test_case) { if (test_case.operator_name().find("q8ta") != std::string::npos) { linear_q8ta_q8csw_reference_impl(test_case); @@ -433,59 +371,7 @@ void reference_impl(TestCase& test_case) { } } -int64_t quantized_linear_flop_calculator(const TestCase& test_case) { - int input_idx = 0; - int weight_idx = 1; - if (test_case.operator_name().find("q8ta") != std::string::npos) { - input_idx = 0; - weight_idx = 3; - } - - // Get input and weight dimensions - const auto& input_sizes = test_case.inputs()[input_idx].get_tensor_sizes(); - const auto& weight_sizes = test_case.inputs()[weight_idx].get_tensor_sizes(); - - int64_t batch_size = input_sizes[0]; - int64_t in_features = input_sizes[1]; - int64_t out_features = weight_sizes[0]; - - // Calculate FLOPs for quantized linear operation - // Each output element requires: - // - in_features multiply-accumulate operations - // - Additional operations for quantization/dequantization - int64_t output_elements = batch_size * out_features; - int64_t ops_per_output = in_features; - - // Add quantization overhead (approximate) - // - Dequantize input: 1 op per input element used - // - Dequantize weight: 1 op per weight element used - // - Add bias: 1 op per output element - int64_t quantization_ops = ops_per_output + 1; // Simplified estimate - - int64_t flop = output_elements * (ops_per_output + quantization_ops); - - return flop; -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Quantized Linear Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_quantized_linear_test_cases, - quantized_linear_flop_calculator, - "QuantizedLinear", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8csw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/q8csw_linear.h b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/q8csw_linear.h new file mode 100644 index 00000000000..3a3df55ad63 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/q8csw_linear.h @@ -0,0 +1,43 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_linear { + +constexpr int64_t kRefDimSizeLimit = 300; + +// Linear configuration struct +struct LinearConfig { + int64_t M; // Batch size / number of rows in input + int64_t K; // Input features / columns in input, rows in weight + int64_t N; // Output features / columns in weight + bool has_bias = true; + std::string test_case_name = "placeholder"; + std::string op_name = "linear_q8ta_q8csw"; +}; + +TestCase create_test_case_from_config( + const LinearConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype); + +void reference_impl(TestCase& test_case); + +int64_t quantized_linear_flop_calculator(const TestCase& test_case); + +} // namespace q8csw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/debug_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/debug_set.cpp new file mode 100644 index 00000000000..5402016b048 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/debug_set.cpp @@ -0,0 +1,56 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_linear { + +namespace { + +std::vector generate_single_op_test_cases( + const std::vector& configs, + const std::vector& storage_types, + const std::vector& dtypes) { + std::vector test_cases; + + for (const auto& config : configs) { + for (const auto& storage_type : storage_types) { + for (const auto& input_dtype : dtypes) { + test_cases.push_back( + create_test_case_from_config(config, storage_type, input_dtype)); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8csw_linear", "debug") { + const std::vector configs = { + // Single simple configuration for debugging + {4, 4, 4, true, "simple"}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector dtypes = {vkapi::kFloat}; + + return { + generate_single_op_test_cases(configs, storage_types, dtypes), + reference_impl, + quantized_linear_flop_calculator}; +} + +} // namespace q8csw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/q8csw_linear_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/q8csw_linear_sets.cpp new file mode 100644 index 00000000000..4cbea88deb9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/q8csw_linear_sets.cpp @@ -0,0 +1,98 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_linear { + +namespace { + +std::vector generate_test_cases( + const std::vector& configs, + const std::vector& storage_types) { + std::vector test_cases; + + for (auto config : configs) { + std::string prefix = + (config.M < kRefDimSizeLimit && config.K < kRefDimSizeLimit && + config.N < kRefDimSizeLimit) + ? "correctness_" + : "performance_"; + std::string generated_test_case_name = prefix + std::to_string(config.M) + + "_" + std::to_string(config.K) + "_" + std::to_string(config.N); + if (!config.has_bias) { + generated_test_case_name += "_no_bias"; + } + + config.test_case_name = generated_test_case_name; + + for (const auto& storage_type : storage_types) { + if (vkcompute::api::context() + ->adapter_ptr() + ->supports_int8_dot_product()) { + // Test both activation+weight quantized and weight only quantized + test_cases.push_back( + create_test_case_from_config(config, storage_type, vkapi::kFloat)); + } + + LinearConfig wo_quant_config = config; + wo_quant_config.op_name = "linear_q8csw"; + test_cases.push_back(create_test_case_from_config( + wo_quant_config, storage_type, vkapi::kFloat)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8csw_linear", "correctness") { + const std::vector configs = { + {4, 64, 32}, + {4, 128, 64}, + {4, 256, 128}, + {32, 64, 32}, + {32, 128, 64}, + {32, 256, 128}, + // No bias tests + {32, 128, 64, false}, + {32, 256, 128, false}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + + return { + generate_test_cases(configs, storage_types), + reference_impl, + quantized_linear_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8csw_linear", "performance") { + const std::vector configs = { + {256, 2048, 2048}, + {512, 2048, 2048}, + {1024, 2048, 2048}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + + return { + generate_test_cases(configs, storage_types), + reference_impl, + quantized_linear_flop_calculator}; +} + +} // namespace q8csw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/test_q8ta_binary.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/binary.cpp similarity index 60% rename from backends/vulkan/test/ops/benchmark/cases/q8ta/binary/test_q8ta_binary.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/binary/binary.cpp index 47b65fa2f1e..7c878a2b485 100644 --- a/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/test_q8ta_binary.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/binary.cpp @@ -4,28 +4,19 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include -#include +#include + #include #include -#include -#include -#include - -using namespace executorch::vulkan::prototyping; +#include +#include -static constexpr int64_t kRefDimSizeLimit = 512; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_binary { -// Configuration struct for q8ta binary testing -struct Q8taBinaryConfig { - std::vector shape; // Tensor shape (can be any dimensionality) - std::string test_case_name = "placeholder"; - std::string op_name = "q8ta_add"; - // Shape of input B; empty means same as `shape`. Inputs are broadcast - // against each other. - std::vector other_shape = {}; -}; +namespace { std::vector broadcast_sizes( const std::vector& a, @@ -39,8 +30,6 @@ std::vector broadcast_sizes( return out; } -// Maps a contiguous index into a tensor of out_sizes to the contiguous index -// of the element it reads from a tensor of `sizes` broadcast to out_sizes. int64_t broadcast_src_idx( int64_t out_idx, const std::vector& out_sizes, @@ -62,6 +51,8 @@ int64_t broadcast_src_idx( return src_idx; } +} // namespace + // Utility function to create a test case from a Q8taBinaryConfig TestCase create_test_case_from_config( const Q8taBinaryConfig& config, @@ -69,7 +60,7 @@ TestCase create_test_case_from_config( vkapi::ScalarType input_dtype, utils::GPUMemoryLayout fp_memory_layout, utils::GPUMemoryLayout quant_layout, - bool const_b = false) { + bool const_b) { TestCase test_case; const std::vector& other_shape = @@ -180,161 +171,6 @@ TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for q8ta_add operation (for debugging) -std::vector generate_q8ta_add_easy_cases() { - std::vector test_cases; - - std::vector> shapes = { - {1, 16, 16, 16}, // 4D: [N, C, H, W] - {1, 144}, // 2D: exercises block config with ndim < 4 - {1, 90}, // 2D: matches skin_seg model's keypoint/bbox tensor sizes - }; - - // Quantized memory layouts to test - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - for (const auto& shape : shapes) { - Q8taBinaryConfig config = {shape, "ACCU"}; - for (const auto& quant_layout : quant_layouts) { - test_cases.push_back(create_test_case_from_config( - config, - /*storage_type=*/utils::kBuffer, - /*input_dtype=*/vkapi::kFloat, - /*fp_memory_layout=*/utils::kWidthPacked, - quant_layout)); - test_cases.push_back(create_test_case_from_config( - config, - /*fp_storage_type=*/utils::kBuffer, - /*input_dtype=*/vkapi::kFloat, - /*fp_layout=*/utils::kWidthPacked, - quant_layout, - /*const_b=*/true)); - } - } - - return test_cases; -} - -// Generate test cases for q8ta_add operation -std::vector generate_q8ta_add_test_cases() { - std::vector test_cases; - - // Shapes to test - std::vector> shapes = { - // 1D tensors - {144}, - {90}, - - // 3D tensors - {1, 16, 32}, - {1, 3, 64}, - - // 2D tensors (exercises block config with ndim < 4) - {1, 144}, - {1, 90}, - {1, 4}, - {3, 32}, - - // Small test cases for correctness - {1, 3, 16, 16}, - {1, 8, 32, 32}, - {1, 16, 24, 24}, - {1, 32, 12, 12}, - {1, 1, 64, 64}, - {1, 3, 64, 64}, - {1, 4, 16, 16}, - - // Different tensor sizes - {1, 8, 20, 20}, - {1, 16, 14, 14}, - {1, 8, 28, 28}, - - // Odd tensor sizes - {1, 3, 15, 15}, - {1, 13, 31, 31}, - {1, 17, 23, 23}, - - // Performance test cases (larger tensors) - {1, 64, 128, 128}, - {1, 32, 64, 64}, - {1, 128, 56, 56}, - {1, 128, 128, 128}, - }; - - // Quantized memory layouts to test - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - std::vector configs; - for (const auto& shape : shapes) { - // Generate test case name prefix from shape dimensions - std::string prefix = "ACCU"; - for (const auto& dim : shape) { - if (dim > kRefDimSizeLimit) { - prefix = "PERF"; - break; - } - } - Q8taBinaryConfig config; - config.shape = shape; - config.test_case_name = prefix; - configs.push_back(config); - } - - // Broadcast cases: {input A shape, input B shape} - std::vector, std::vector>> - broadcast_shapes = { - // Per-video context row added to every frame row - {{1, 60, 512}, {1, 1, 512}}, - {{1, 1, 512}, {1, 60, 512}}, - {{1, 16, 32}, {1, 16, 1}}, - {{1, 16, 32}, {32}}, - {{1, 8, 16, 16}, {1, 8, 1, 1}}, - {{1, 8, 16, 16}, {1, 1, 16, 16}}, - {{2, 8, 6, 6}, {1, 8, 6, 6}}, - {{1, 13, 7, 9}, {1, 1, 7, 1}}, - }; - for (const auto& [shape, other_shape] : broadcast_shapes) { - Q8taBinaryConfig config; - config.shape = shape; - config.test_case_name = "ACCU"; - config.other_shape = other_shape; - configs.push_back(config); - } - - // Generate all combinations - for (const auto& config : configs) { - for (const auto& quant_layout : quant_layouts) { - test_cases.push_back(create_test_case_from_config( - config, - /*storage_type=*/utils::kBuffer, - /*input_dtype=*/vkapi::kFloat, - /*fp_memory_layout=*/utils::kWidthPacked, - quant_layout)); - test_cases.push_back(create_test_case_from_config( - config, - /*fp_storage_type=*/utils::kBuffer, - /*fp_input_dtype=*/vkapi::kFloat, - /*fp_layout=*/utils::kWidthPacked, - quant_layout, - /*const_b=*/true)); - } - } - - return test_cases; -} - // Reference implementation for quantized add operation void q8ta_add_reference_impl(TestCase& test_case) { int32_t idx = 0; @@ -440,32 +276,7 @@ void q8ta_add_reference_impl(TestCase& test_case) { } } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(false); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Q8TA Binary Add Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = q8ta_add_reference_impl; - - auto results = execute_test_cases( -#ifdef DEBUG_MODE - generate_q8ta_add_easy_cases, -#else - generate_q8ta_add_test_cases, -#endif - "Q8taBinaryAdd", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_binary +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/binary.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/binary.h new file mode 100644 index 00000000000..5e5541397bf --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/binary.h @@ -0,0 +1,45 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_binary { + +constexpr int64_t kRefDimSizeLimit = 512; + +// Configuration struct for q8ta binary testing +struct Q8taBinaryConfig { + std::vector shape; // Tensor shape (can be any dimensionality) + std::string test_case_name = "placeholder"; + std::string op_name = "q8ta_add"; + // Shape of input B; empty means same as `shape`. Inputs are broadcast + // against each other. + std::vector other_shape = {}; +}; + +TestCase create_test_case_from_config( + const Q8taBinaryConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype, + utils::GPUMemoryLayout fp_memory_layout, + utils::GPUMemoryLayout quant_layout, + bool const_b = false); + +void q8ta_add_reference_impl(TestCase& test_case); + +} // namespace q8ta_binary +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/test_sets/binary_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/test_sets/binary_sets.cpp new file mode 100644 index 00000000000..6470f4f7373 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/test_sets/binary_sets.cpp @@ -0,0 +1,168 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_binary { + +namespace { + +std::vector generate_test_cases( + const std::vector>& shapes, + const std::vector, std::vector>>& + broadcast_shapes, + const std::vector& quant_layouts) { + std::vector test_cases; + + std::vector configs; + for (const auto& shape : shapes) { + // Generate test case name prefix from shape dimensions + std::string prefix = "ACCU"; + for (const auto& dim : shape) { + if (dim > kRefDimSizeLimit) { + prefix = "PERF"; + break; + } + } + + Q8taBinaryConfig config; + config.shape = shape; + config.test_case_name = prefix; + configs.push_back(config); + } + for (const auto& [shape, other_shape] : broadcast_shapes) { + Q8taBinaryConfig config; + config.shape = shape; + config.test_case_name = "ACCU"; + config.other_shape = other_shape; + configs.push_back(config); + } + + // Generate all combinations + for (const auto& config : configs) { + for (const auto& quant_layout : quant_layouts) { + test_cases.push_back(create_test_case_from_config( + config, + /*storage_type=*/utils::kBuffer, + /*input_dtype=*/vkapi::kFloat, + /*fp_memory_layout=*/utils::kWidthPacked, + quant_layout)); + test_cases.push_back(create_test_case_from_config( + config, + /*storage_type=*/utils::kBuffer, + /*input_dtype=*/vkapi::kFloat, + /*fp_memory_layout=*/utils::kWidthPacked, + quant_layout, + /*const_b=*/true)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/binary", "correctness") { + // Shapes to test + const std::vector> shapes = { + // 1D tensors + {144}, + {90}, + + // 3D tensors + {1, 16, 32}, + {1, 3, 64}, + + // 2D tensors (exercises block config with ndim < 4) + {1, 144}, + {1, 90}, + {1, 4}, + {3, 32}, + + // Small test cases for correctness + {1, 3, 16, 16}, + {1, 8, 32, 32}, + {1, 16, 24, 24}, + {1, 32, 12, 12}, + {1, 1, 64, 64}, + {1, 3, 64, 64}, + {1, 4, 16, 16}, + + // Different tensor sizes + {1, 8, 20, 20}, + {1, 16, 14, 14}, + {1, 8, 28, 28}, + + // Odd tensor sizes + {1, 3, 15, 15}, + {1, 13, 31, 31}, + {1, 17, 23, 23}, + + // Performance test cases (larger tensors) + {1, 64, 128, 128}, + {1, 32, 64, 64}, + {1, 128, 56, 56}, + {1, 128, 128, 128}, + }; + // Quantized memory layouts to test + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + // Broadcast cases: {input A shape, input B shape} + const std::vector, std::vector>> + broadcast_shapes = { + // Per-video context row added to every frame row + {{1, 60, 512}, {1, 1, 512}}, + {{1, 1, 512}, {1, 60, 512}}, + {{1, 16, 32}, {1, 16, 1}}, + {{1, 16, 32}, {32}}, + {{1, 8, 16, 16}, {1, 8, 1, 1}}, + {{1, 8, 16, 16}, {1, 1, 16, 16}}, + {{2, 8, 6, 6}, {1, 8, 6, 6}}, + {{1, 13, 7, 9}, {1, 1, 7, 1}}, + }; + return { + generate_test_cases(shapes, broadcast_shapes, quant_layouts), + q8ta_add_reference_impl}; +} + +// Easy test cases for q8ta_add operation (for debugging) +REGISTER_TEST_CASE_SET("q8ta/binary", "debug") { + const std::vector> shapes = { + {1, 16, 16, 16}, // 4D: [N, C, H, W] + {1, 144}, // 2D: exercises block config with ndim < 4 + {1, 90}, // 2D: matches skin_seg model's keypoint/bbox tensor sizes + }; + // Quantized memory layouts to test + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + const std::vector, std::vector>> + broadcast_shapes = {}; + return { + generate_test_cases(shapes, broadcast_shapes, quant_layouts), + q8ta_add_reference_impl}; +} + +} // namespace q8ta_binary +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_q8ta_clone.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/clone.cpp similarity index 52% rename from backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_q8ta_clone.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/clone/clone.cpp index 2f6ea11b6c8..7ed1363695c 100644 --- a/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_q8ta_clone.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/clone.cpp @@ -4,29 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include -#include +#include + #include #include -#include +#include +#include #include -#include - -// #define DEBUG_MODE - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 512; - -// Configuration struct for q8ta clone testing -struct Q8taCloneConfig { - std::vector shape; // Tensor shape (can be any dimensionality) - std::string test_case_name = "placeholder"; - std::string op_name = "q8ta_clone_test"; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_clone { // Utility function to create a test case from a Q8taCloneConfig TestCase create_test_case_from_config( @@ -109,140 +98,6 @@ TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for q8ta_clone operation (for debugging) -std::vector generate_q8ta_clone_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - Q8taCloneConfig config = { - {1, 16, 16, 16}, // shape: [N, C, H, W] - "ACCU", // test_case_name - }; - - // FP memory layouts to test - std::vector fp_layouts = { - utils::kWidthPacked, - utils::kChannelsPacked, - }; - - // Quantized memory layouts to test - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - std::vector storage_types = {utils::kBuffer}; - std::vector float_types = {vkapi::kFloat}; - - // Generate test cases for each combination (same layout for input and output) - for (const auto& fp_layout : fp_layouts) { - for (const auto& quant_layout : quant_layouts) { - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - // Same layout: should be a simple copy - test_cases.push_back(create_test_case_from_config( - config, - storage_type, - input_dtype, - fp_layout, - quant_layout, - quant_layout)); - } - } - } - } - - return test_cases; -} - -// Generate test cases for q8ta_clone operation -std::vector generate_q8ta_clone_test_cases() { - std::vector test_cases; - - // Shapes to test - std::vector> shapes = { - // Small test cases for correctness - {1, 3, 16, 16}, - {1, 8, 32, 32}, - {1, 16, 24, 24}, - {1, 32, 12, 12}, - {1, 1, 64, 64}, - {1, 3, 64, 64}, - {1, 4, 16, 16}, - - // Different tensor sizes - {1, 8, 20, 20}, - {1, 16, 14, 14}, - {1, 8, 28, 28}, - - // Odd tensor sizes - {1, 3, 15, 15}, - {1, 13, 31, 31}, - {1, 17, 23, 23}, - - // Performance test cases (larger tensors) - {1, 64, 128, 128}, - {1, 32, 64, 64}, - {1, 128, 56, 56}, - {1, 128, 128, 128}, - }; - - // FP memory layouts to test - std::vector fp_layouts = { - utils::kWidthPacked, - utils::kChannelsPacked, - }; - - // Quantized memory layouts to test - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - // Test with buffer storage only - std::vector storage_types = {utils::kBuffer}; - - // Generate all combinations - for (const auto& shape : shapes) { - // Generate test case name prefix from shape dimensions - std::string prefix = "ACCU"; - for (const auto& dim : shape) { - if (dim > kRefDimSizeLimit) { - prefix = "PERF"; - break; - } - } - - for (const auto& fp_layout : fp_layouts) { - for (const auto& inp_quant_layout : quant_layouts) { - for (const auto& outp_quant_layout : quant_layouts) { - for (const auto& storage_type : storage_types) { - Q8taCloneConfig config; - config.shape = shape; - config.test_case_name = prefix; - - test_cases.push_back(create_test_case_from_config( - config, - storage_type, - vkapi::kFloat, - fp_layout, - inp_quant_layout, - outp_quant_layout)); - } - } - } - } - } - - return test_cases; -} - // Reference implementation for q8ta_clone operation // Since clone just copies data, the result should be the same as // quantize-dequantize @@ -315,32 +170,18 @@ void q8ta_clone_reference_impl(TestCase& test_case) { } } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(false); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Q8TA Clone Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = q8ta_clone_reference_impl; - - auto results = execute_test_cases( -#ifdef DEBUG_MODE - generate_q8ta_clone_easy_cases, -#else - generate_q8ta_clone_test_cases, -#endif - "Q8taClone", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; +std::string get_prefix(const std::vector& shape) { + std::string prefix = "ACCU"; + for (const auto& dim : shape) { + if (dim > kRefDimSizeLimit) { + prefix = "PERF"; + break; + } + } + return prefix; } + +} // namespace q8ta_clone +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/clone.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/clone.h new file mode 100644 index 00000000000..2e0677fd646 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/clone.h @@ -0,0 +1,45 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_clone { + +constexpr int64_t kRefDimSizeLimit = 512; + +// Configuration struct for q8ta clone testing +struct Q8taCloneConfig { + std::vector shape; // Tensor shape (can be any dimensionality) + std::string test_case_name = "placeholder"; + std::string op_name = "q8ta_clone_test"; +}; + +TestCase create_test_case_from_config( + const Q8taCloneConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype, + utils::GPUMemoryLayout fp_memory_layout, + utils::GPUMemoryLayout inp_quant_layout, + utils::GPUMemoryLayout outp_quant_layout); + +void q8ta_clone_reference_impl(TestCase& test_case); + +// "PERF" if any dim of shape exceeds kRefDimSizeLimit, else "ACCU". +std::string get_prefix(const std::vector& shape); + +} // namespace q8ta_clone +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/correctness_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/correctness_set.cpp new file mode 100644 index 00000000000..509f249672b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/correctness_set.cpp @@ -0,0 +1,107 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_clone { + +namespace { + +std::vector generate_test_cases( + const std::vector>& shapes, + const std::vector& fp_layouts, + const std::vector& quant_layouts, + const std::vector& storage_types) { + std::vector test_cases; + + // Generate all combinations + for (const auto& shape : shapes) { + std::string prefix = get_prefix(shape); + + for (const auto& fp_layout : fp_layouts) { + for (const auto& inp_quant_layout : quant_layouts) { + for (const auto& outp_quant_layout : quant_layouts) { + for (const auto& storage_type : storage_types) { + Q8taCloneConfig config; + config.shape = shape; + config.test_case_name = prefix; + + test_cases.push_back(create_test_case_from_config( + config, + storage_type, + vkapi::kFloat, + fp_layout, + inp_quant_layout, + outp_quant_layout)); + } + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/clone", "correctness") { + // Shapes to test + const std::vector> shapes = { + // Small test cases for correctness + {1, 3, 16, 16}, + {1, 8, 32, 32}, + {1, 16, 24, 24}, + {1, 32, 12, 12}, + {1, 1, 64, 64}, + {1, 3, 64, 64}, + {1, 4, 16, 16}, + + // Different tensor sizes + {1, 8, 20, 20}, + {1, 16, 14, 14}, + {1, 8, 28, 28}, + + // Odd tensor sizes + {1, 3, 15, 15}, + {1, 13, 31, 31}, + {1, 17, 23, 23}, + + // Performance test cases (larger tensors) + {1, 64, 128, 128}, + {1, 32, 64, 64}, + {1, 128, 56, 56}, + {1, 128, 128, 128}, + }; + // FP memory layouts to test + const std::vector fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + // Quantized memory layouts to test + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + // Test with buffer storage only + const std::vector storage_types = {utils::kBuffer}; + return { + generate_test_cases(shapes, fp_layouts, quant_layouts, storage_types), + q8ta_clone_reference_impl}; +} + +} // namespace q8ta_clone +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/debug_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/debug_set.cpp new file mode 100644 index 00000000000..97f869ab02e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/debug_set.cpp @@ -0,0 +1,84 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_clone { + +namespace { + +std::vector generate_same_layout_test_cases( + const std::vector>& shapes, + const std::vector& fp_layouts, + const std::vector& quant_layouts, + const std::vector& storage_types, + const std::vector& float_types) { + std::vector test_cases; + + for (const auto& shape : shapes) { + Q8taCloneConfig config = {shape, get_prefix(shape)}; + + // Generate test cases for each combination (same layout for input and + // output) + for (const auto& fp_layout : fp_layouts) { + for (const auto& quant_layout : quant_layouts) { + for (const auto& storage_type : storage_types) { + for (const auto& input_dtype : float_types) { + // Same layout: should be a simple copy + test_cases.push_back(create_test_case_from_config( + config, + storage_type, + input_dtype, + fp_layout, + quant_layout, + quant_layout)); + } + } + } + } + } + + return test_cases; +} + +} // namespace + +// Easy test cases for q8ta_clone operation (for debugging) +REGISTER_TEST_CASE_SET("q8ta/clone", "debug") { + // Single simple configuration for debugging + const std::vector> shapes = { + {1, 16, 16, 16}, // shape: [N, C, H, W] + }; + // FP memory layouts to test + const std::vector fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + // Quantized memory layouts to test + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + const std::vector storage_types = {utils::kBuffer}; + const std::vector float_types = {vkapi::kFloat}; + return { + generate_same_layout_test_cases( + shapes, fp_layouts, quant_layouts, storage_types, float_types), + q8ta_clone_reference_impl}; +} + +} // namespace q8ta_clone +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.cpp new file mode 100644 index 00000000000..cd80172a3d3 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.cpp @@ -0,0 +1,605 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +// Utility function to create a test case from a Conv2dConfig +TestCase create_test_case_from_config_with_layouts( + const Conv2dConfig& config, + vkapi::ScalarType input_dtype, + utils::StorageType fp_storage_type, + utils::GPUMemoryLayout input_int8_memory_layout, + utils::GPUMemoryLayout output_int8_memory_layout, + const std::string& impl_selector = "", + const Im2colUnsignedTestOptions* im2col_options = nullptr, + const float input_scale_val = 0.008123f, + const DataGenType input_data_gen = DataGenType::RANDOM) { + TestCase test_case; + + // Calculate output dimensions + int64_t H_out = config.get_output_height(); + int64_t W_out = config.get_output_width(); + + // Input tensor (float/half) - [N, C_in, H_in, W_in] + std::vector input_size = { + config.batch, + config.channels.in, + config.input_size.h, + config.input_size.w}; + + utils::GPUMemoryLayout fp_memory_layout = fp_storage_type == utils::kBuffer + ? utils::kWidthPacked + : utils::kChannelsPacked; + + // Create test case name + std::string prefix = config.test_case_name.substr(0, 4); // "ACCU" or "PERF" + std::string dtype_str = dtype_short(input_dtype); + std::string in_shape = "[" + std::to_string(config.batch) + "," + + std::to_string(config.channels.in) + "," + + std::to_string(config.input_size.h) + "," + + std::to_string(config.input_size.w) + "]"; + std::string weight_shape = "[" + std::to_string(config.channels.out) + "," + + std::to_string(config.channels.in / config.groups) + "," + + std::to_string(config.kernel.h) + "," + std::to_string(config.kernel.w) + + "]"; + std::string shape_str = in_shape + "x" + weight_shape + " s" + + std::to_string(config.stride.h) + " p" + + std::to_string(config.padding.h) + " d" + + std::to_string(config.dilation.h) + " g" + std::to_string(config.groups); + std::string storage_str = repr_str(utils::kBuffer, input_int8_memory_layout); + if (input_int8_memory_layout != output_int8_memory_layout) { + storage_str += "->" + repr_str(utils::kBuffer, output_int8_memory_layout); + } + std::string suffix = impl_selector.empty() ? "" : "[" + impl_selector + "]"; + std::string test_name = make_test_label( + prefix, dtype_str, dtype_str, shape_str, storage_str, suffix); + test_case.set_name(test_name); + + // Set the operator name for the test case - use the unified test operator + std::string operator_name = "test_etvk.test_q8ta_conv2d.default"; + test_case.set_operator_name(operator_name); + + ValueSpec input_tensor( + input_size, + input_dtype, + fp_storage_type, + fp_memory_layout, + input_data_gen); + + if (debugging()) { + print_valuespec_data(input_tensor, "input_tensor"); + } + + ValueSpec input_scale(input_scale_val); + + const int32_t input_zero_point_val = + im2col_options == nullptr ? 2 : im2col_options->input_zero_point; + ValueSpec input_zero_point(input_zero_point_val); + + if (im2col_options != nullptr && + im2col_options->use_accumulator_limit_values) { + input_tensor.ensure_data_generated(2401); + std::fill( + input_tensor.get_float_data().begin(), + input_tensor.get_float_data().end(), + (127.0f - input_zero_point_val) * input_scale_val); + } else if (im2col_options != nullptr && im2col_options->use_extreme_values) { + input_tensor.ensure_data_generated(2401); + constexpr std::array values = {-128, -1, 0, 1, 127}; + std::vector& input_data = input_tensor.get_float_data(); + for (size_t i = 0; i < input_data.size(); ++i) { + input_data.at(i) = (static_cast(values.at(i % values.size())) - + input_zero_point_val) * + input_scale_val; + } + } + + // Quantized weight tensor (int8) - [C_out, C_in_per_group * K_h * K_w] + // Memory layout: height, width, then channels - in_c is innermost (stride 1) + // in the second dimension + const int64_t in_channels_per_group = config.channels.in / config.groups; + const int64_t in_features = utils::align_up_4( + in_channels_per_group * config.kernel.h * config.kernel.w); + std::vector weight_size = {config.channels.out, in_features}; + ValueSpec quantized_weight( + weight_size, + vkapi::kChar, // int8 for quantized weights + fp_storage_type, + utils::kWidthPacked, + DataGenType::RANDINT8); + quantized_weight.set_constant(true); + + if (im2col_options != nullptr && + im2col_options->use_accumulator_limit_values) { + quantized_weight.ensure_data_generated(2402); + std::fill( + quantized_weight.get_int8_data().begin(), + quantized_weight.get_int8_data().end(), + 127); + } else if (im2col_options != nullptr && im2col_options->use_extreme_values) { + quantized_weight.ensure_data_generated(2402); + constexpr std::array values = {-128, -1, 0, 1, 127}; + std::vector& weight_data = quantized_weight.get_int8_data(); + for (size_t i = 0; i < weight_data.size(); ++i) { + weight_data.at(i) = values.at((i * 3 + 1) % values.size()); + } + } + + if (debugging()) { + print_valuespec_data(quantized_weight, "weight_tensor"); + } + + const int64_t aligned_out_channels = utils::align_up_4(config.channels.out); + + // Weight quantization scales (float/half, per-channel) + ValueSpec weight_scales( + {aligned_out_channels}, // Per output channel + input_dtype, + fp_storage_type, + utils::kWidthPacked, + DataGenType::RANDOM_SCALES); + weight_scales.set_constant(true); + if (im2col_options != nullptr) { + weight_scales.ensure_data_generated(2403); + std::fill( + weight_scales.get_float_data().begin(), + weight_scales.get_float_data().end(), + im2col_options->weight_scale); + } + + ValueSpec weight_sums( + {aligned_out_channels}, // Per output channel + vkapi::kInt, + fp_storage_type, + utils::kWidthPacked, + DataGenType::ZEROS); + weight_sums.set_constant(true); + + // Compute weight_sums data based on quantized weights + compute_weight_sums( + weight_sums, quantized_weight, config.channels.out, in_features); + + // Bias (optional, float/half) - [C_out] + ValueSpec bias( + {aligned_out_channels}, // Per output channel + input_dtype, + fp_storage_type, + utils::kWidthPacked, + DataGenType::ZEROS); + bias.set_constant(true); + if (im2col_options != nullptr && !im2col_options->has_bias) { + bias.set_none(true); + } + + // Output quantization parameters + float output_scale_val = 0.05314; + ValueSpec output_scale(output_scale_val); + + const int32_t output_zero_point_val = + im2col_options == nullptr ? -1 : im2col_options->output_zero_point; + ValueSpec output_zero_point(output_zero_point_val); + + // Stride and padding parameters + ValueSpec stride({config.stride.h, config.stride.w}); + ValueSpec padding({config.padding.h, config.padding.w}); + + // Dilation and groups parameters + ValueSpec dilation({config.dilation.h, config.dilation.w}); + ValueSpec groups(config.groups); + + // Kernel size parameters + ValueSpec kernel_size({config.kernel.h, config.kernel.w}); + + // Output tensor (float/half) - [N, C_out, H_out, W_out] + ValueSpec output( + {config.batch, config.channels.out, H_out, W_out}, + input_dtype, + fp_storage_type, + fp_memory_layout, + DataGenType::ZEROS); + + // Add all specs to test case for q8ta_q8csw_q8to operation + test_case.add_input_spec(input_tensor); + test_case.add_input_spec(input_scale); + test_case.add_input_spec(input_zero_point); + test_case.add_input_spec(quantized_weight); + test_case.add_input_spec(weight_sums); + test_case.add_input_spec(weight_scales); + test_case.add_input_spec(output_scale); + test_case.add_input_spec(output_zero_point); + test_case.add_input_spec(bias); + test_case.add_input_spec(kernel_size); + test_case.add_input_spec(stride); + test_case.add_input_spec(padding); + test_case.add_input_spec(dilation); + test_case.add_input_spec(groups); + + // Activation (none = no activation) + ValueSpec activation = ValueSpec::make_string( + im2col_options == nullptr ? "none" : im2col_options->activation); + test_case.add_input_spec(activation); + + // Add memory layout parameter for the quantized tensors + ValueSpec input_layout_int(static_cast(input_int8_memory_layout)); + test_case.add_input_spec(input_layout_int); + + ValueSpec output_layout_int(static_cast(output_int8_memory_layout)); + test_case.add_input_spec(output_layout_int); + + // Add impl_selector string + ValueSpec impl_selector_spec = ValueSpec::make_string(impl_selector); + test_case.add_input_spec(impl_selector_spec); + + test_case.add_output_spec(output); + + test_case.set_abs_tolerance( + im2col_options == nullptr ? output_scale_val + 1e-4f + : output_scale_val * 0.25f); + + // Filter out quantize/dequantize shaders from timing measurements + test_case.set_shader_filter({ + "nchw_to", + "to_nchw", + "q8ta_quantize", + "q8ta_dequantize", + }); + + return test_case; +} + +} // namespace + +TestCase create_test_case_from_config( + const Conv2dConfig& config, + vkapi::ScalarType input_dtype, + utils::StorageType fp_storage_type, + utils::GPUMemoryLayout int8_memory_layout, + const std::string& impl_selector, + const Im2colUnsignedTestOptions* im2col_options, + const float input_scale_val, + const DataGenType input_data_gen) { + return create_test_case_from_config_with_layouts( + config, + input_dtype, + fp_storage_type, + int8_memory_layout, + int8_memory_layout, + impl_selector, + im2col_options, + input_scale_val, + input_data_gen); +} + +// SceneX route tests. The kPackedInt8_4C input + kPackedInt8_4W4C output +// layout combination is only exercised here (default generators never pair +// them); run via the scenex_regular and scenex_grouped sets. Zero +// tolerances are exact by construction: both pipelines accumulate in int32 +// with identical requantize, so any mismatch is a real regression, not noise. +TestCase create_scenex_test_case( + const Conv2dConfig& source_config, + const std::string& route) { + Conv2dConfig config = source_config; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = + make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); + + const std::string impl_selector = route == "auto" ? "" + : route == "direct" ? "general" + : "im2col"; + TestCase test_case = create_test_case_from_config_with_layouts( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + impl_selector, + /*im2col_options=*/nullptr, + /*input_scale_val=*/1.0f, + /*input_data_gen=*/DataGenType::RANDINT); + test_case.set_abs_tolerance(0.0f); + test_case.set_rel_tolerance(0.0f); + return test_case; +} + +// Custom FLOP calculator for quantized conv2d operation +int64_t quantized_conv2d_flop_calculator(const TestCase& test_case) { + int kernel_idx = 9; // kernel_size is at index 9 for q8ta_q8csw_q8to + + // Get input and weight dimensions + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); + + const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); + + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t C_out = output_sizes[1]; + int64_t K_h = kernel_sizes[0]; + int64_t K_w = kernel_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; + + // Calculate FLOPs for quantized conv2d operation + // Each output element requires: + // - C_in * K_h * K_w multiply-accumulate operations + // - Additional operations for quantization/dequantization + int64_t output_elements = N * C_out * H_out * W_out; + int64_t ops_per_output = C_in * K_h * K_w; + + int64_t flop = output_elements * (ops_per_output); + + return flop; +} + +namespace { + +// Reference implementation for activation, weight, and output quantized conv2d +void conv2d_q8ta_q8csw_q8to_reference_impl(TestCase& test_case) { + // Extract input specifications + int32_t idx = 0; + const ValueSpec& input_spec = test_case.inputs()[idx++]; + const ValueSpec& input_scale_spec = test_case.inputs()[idx++]; + const ValueSpec& input_zeros_spec = test_case.inputs()[idx++]; + const ValueSpec& weight_spec = test_case.inputs()[idx++]; + const ValueSpec& weight_sums_spec = test_case.inputs()[idx++]; + (void)weight_sums_spec; + const ValueSpec& weight_scales_spec = test_case.inputs()[idx++]; + const ValueSpec& output_scale_spec = test_case.inputs()[idx++]; + const ValueSpec& output_zeros_spec = test_case.inputs()[idx++]; + const ValueSpec& bias_spec = test_case.inputs()[idx++]; + const ValueSpec& kernel_size_spec = test_case.inputs()[idx++]; + const ValueSpec& stride_spec = test_case.inputs()[idx++]; + const ValueSpec& padding_spec = test_case.inputs()[idx++]; + const ValueSpec& dilation_spec = test_case.inputs()[idx++]; + const ValueSpec& groups_spec = test_case.inputs()[idx++]; + const ValueSpec& activation_spec = test_case.inputs()[idx++]; + const ValueSpec& layout_spec = test_case.inputs()[idx++]; + (void)layout_spec; // Not used in reference implementation + const ValueSpec& output_layout_spec = test_case.inputs()[idx++]; + (void)output_layout_spec; // Not used in reference implementation + const ValueSpec& impl_selector_spec = test_case.inputs()[idx++]; + (void)impl_selector_spec; // Not used in reference implementation + + // Extract output specification (mutable reference) + ValueSpec& output_spec = test_case.outputs()[0]; + + // Get tensor dimensions + auto input_sizes = input_spec.get_tensor_sizes(); // [N, C_in, H_in, W_in] + auto weight_sizes = + weight_spec.get_tensor_sizes(); // [C_out, C_in_per_group * K_h * K_w] + auto output_sizes = + output_spec.get_tensor_sizes(); // [N, C_out, H_out, W_out] + + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t H_in = input_sizes[2]; + int64_t W_in = input_sizes[3]; + int64_t C_out = output_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; + + // Get kernel dimensions from kernel_size ValueSpec + auto kernel_size_data = kernel_size_spec.get_int32_data(); + int64_t K_h = kernel_size_data[0]; + int64_t K_w = kernel_size_data[1]; + + // Get stride, padding, dilation, and groups + auto stride_data = stride_spec.get_int32_data(); + auto padding_data = padding_spec.get_int32_data(); + auto dilation_data = dilation_spec.get_int32_data(); + int64_t stride_h = stride_data[0]; + int64_t stride_w = stride_data[1]; + int64_t pad_h = padding_data[0]; + int64_t pad_w = padding_data[1]; + int64_t dilation_h = dilation_data[0]; + int64_t dilation_w = dilation_data[1]; + int64_t groups = groups_spec.get_int_value(); + + const int64_t reference_operations = + N * C_out * H_out * W_out * (C_in / groups) * K_h * K_w; + const bool has_large_dimension = N > kRefDimSizeLimit || + C_in > kRefDimSizeLimit || H_in > kRefDimSizeLimit || + W_in > kRefDimSizeLimit || C_out > kRefDimSizeLimit; + if (has_large_dimension && reference_operations > kRefOperationLimit) { + throw std::invalid_argument( + "One or more dimensions exceed the allowed limit for reference implementation."); + std::cout + << "Reference implementation: computation may take some time for large tensors..." + << std::endl; + } + + if (input_spec.dtype != vkapi::kFloat) { + throw std::invalid_argument("Unsupported dtype"); + } + + // Get raw data pointers + auto& input_data = input_spec.get_float_data(); + const float input_scale = input_scale_spec.get_float_value(); + const int32_t input_zero_point = input_zeros_spec.get_int_value(); + + auto& weight_data = weight_spec.get_int8_data(); + auto& weight_scales_data = weight_scales_spec.get_float_data(); + auto& bias_data = bias_spec.get_float_data(); + + const float output_scale = output_scale_spec.get_float_value(); + const int32_t output_zero_point = output_zeros_spec.get_int_value(); + + // Calculate channels per group for grouped convolution + int64_t C_in_per_group = C_in / groups; + int64_t C_out_per_group = C_out / groups; + + // Calculate number of output elements + int64_t num_output_elements = N * C_out * H_out * W_out; + + auto& ref_data = output_spec.get_ref_float_data(); + ref_data.resize(num_output_elements); + + const int64_t in_features = utils::align_up_4(C_in_per_group * K_h * K_w); + + // Perform activation, weight, and output quantized conv2d operation + for (int64_t n = 0; n < N; ++n) { + for (int64_t out_c = 0; out_c < C_out; ++out_c) { + for (int64_t out_h = 0; out_h < H_out; ++out_h) { + for (int64_t out_w = 0; out_w < W_out; ++out_w) { + int32_t int_sum = 0; + int32_t weight_sum = 0; // Track weight sum on the fly + + // Determine which group this output channel belongs to + int64_t group_idx = out_c / C_out_per_group; + int64_t in_c_start = group_idx * C_in_per_group; + int64_t in_c_end = (group_idx + 1) * C_in_per_group; + + // Convolution operation with integer accumulation + for (int64_t in_c = in_c_start; in_c < in_c_end; ++in_c) { + for (int64_t kh = 0; kh < K_h; ++kh) { + for (int64_t kw = 0; kw < K_w; ++kw) { + // Calculate input position with dilation + int64_t in_h = out_h * stride_h - pad_h + kh * dilation_h; + int64_t in_w = out_w * stride_w - pad_w + kw * dilation_w; + + // Check bounds (zero padding) + if (in_h >= 0 && in_h < H_in && in_w >= 0 && in_w < W_in) { + // Get input value and quantize to int8 + int64_t input_idx = n * (C_in * H_in * W_in) + + in_c * (H_in * W_in) + in_h * W_in + in_w; + + float quant_input_f = + std::round(input_data[input_idx] / input_scale) + + input_zero_point; + quant_input_f = + std::min(std::max(quant_input_f, -128.0f), 127.0f); + int8_t quantized_input = static_cast(quant_input_f); + + // Get quantized weight (already int8) + // Weight layout: [C_out, C_in_per_group * K_h * K_w] + int64_t weight_idx = out_c * in_features + + (kh * (K_w * C_in_per_group) + kw * C_in_per_group + + (in_c % C_in_per_group)); + int8_t quantized_weight = weight_data[weight_idx]; + + // Integer multiplication and accumulation + int_sum += static_cast(quantized_input) * + static_cast(quantized_weight); + + // Track weight sum for this output channel on the fly + weight_sum += static_cast(quantized_weight); + } else { + // For zero padding, we still need to account for the weight + // in weight_sum when input is effectively 0 (but quantized 0 + // is input_zero_point) + int64_t weight_idx = out_c * in_features + + (kh * (K_w * C_in_per_group) + kw * C_in_per_group + + (in_c % C_in_per_group)); + int8_t quantized_weight = weight_data[weight_idx]; + + // Add contribution from zero-padded input (quantized zero = + // input_zero_point) + int_sum += static_cast(input_zero_point) * + static_cast(quantized_weight); + + // Track weight sum for this output channel on the fly + weight_sum += static_cast(quantized_weight); + } + } + } + } + + // Convert accumulated integer result to float and apply scales + // Final result = (int_sum - zero_point_correction) * input_scale * + // weight_scale + bias zero_point_correction = input_zero_point * + // sum_of_weights_for_this_output_channel + int32_t zero_point_correction = input_zero_point * weight_sum; + int32_t accum_adjusted = int_sum - zero_point_correction; + float float_result = + accum_adjusted * input_scale * weight_scales_data[out_c]; + + if (!bias_spec.is_none()) { + float_result += bias_data[out_c]; + } + if (activation_spec.get_string_value() == "relu") { + float_result = std::max(float_result, 0.0f); + } + + // Quantize the output to int8 + float quant_output_f = + std::round(float_result / output_scale) + output_zero_point; + quant_output_f = std::min(std::max(quant_output_f, -128.0f), 127.0f); + int8_t quantized_output = static_cast(quant_output_f); + + // Dequantize back to float + float dequant_output = + (static_cast(quantized_output) - output_zero_point) * + output_scale; + + int64_t output_idx = n * (C_out * H_out * W_out) + + out_c * (H_out * W_out) + out_h * W_out + out_w; + ref_data[output_idx] = dequant_output; + } + } + } + } +} + +} // namespace + +void reference_impl(TestCase& test_case) { + conv2d_q8ta_q8csw_q8to_reference_impl(test_case); +} + +// The impl selector holds one of "", "general", or "im2col"; activation and +// other string inputs use disjoint values. Overwrite it by value so a +// reordered input list fails loudly instead of mutating the wrong spec. +// Note: for route == "direct" the measured run already forces "general", so +// this reference re-executes the identical implementation and only checks +// determinism; genuine cross-implementation correctness comes from the +// auto/im2col legs. +void scenex_direct_reference(TestCase& test_case) { + TestCase direct_case = test_case; + bool found_selector = false; + for (auto it = direct_case.inputs().rbegin(); + it != direct_case.inputs().rend(); + ++it) { + if (it->is_string() && + (it->get_string_value().empty() || + it->get_string_value() == "general" || + it->get_string_value() == "im2col")) { + it->string_data = "general"; + found_selector = true; + break; + } + } + if (!found_selector) { + throw std::runtime_error("scenex reference: impl selector input not found"); + } + execute_test_case( + direct_case, + /*warmup_runs=*/1, + /*benchmark_runs=*/1, + /*chained_dispatches=*/1, + /*write_outputs=*/true); + test_case.outputs().at(0).get_ref_float_data() = + direct_case.outputs().at(0).get_float_data(); +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.h new file mode 100644 index 00000000000..7d87b00f63f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.h @@ -0,0 +1,81 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +constexpr int64_t kRefDimSizeLimit = 100; +constexpr int64_t kRefOperationLimit = 2 * 1024 * 1024; + +struct Im2colUnsignedTestOptions { + int32_t input_zero_point = 2; + int32_t output_zero_point = -1; + bool use_extreme_values = false; + bool use_accumulator_limit_values = false; + bool has_bias = true; + const char* activation = "none"; + float weight_scale = 1.0f / 256.0f; +}; + +struct PointwiseTestOptions { + int32_t input_zero_point = 2; + bool has_bias = true; +}; + +TestCase create_test_case_from_config( + const Conv2dConfig& config, + vkapi::ScalarType input_dtype, + utils::StorageType fp_storage_type, + utils::GPUMemoryLayout int8_memory_layout, + const std::string& impl_selector = "", + const Im2colUnsignedTestOptions* im2col_options = nullptr, + const float input_scale_val = 0.008123f, + const DataGenType input_data_gen = DataGenType::RANDOM); + +TestCase create_scenex_test_case( + const Conv2dConfig& source_config, + const std::string& route); + +TestCase create_dw_test_case_from_config( + const Conv2dConfig& config, + vkapi::ScalarType input_dtype, + utils::StorageType fp_storage_type, + utils::GPUMemoryLayout int8_memory_layout, + const std::string& impl_selector = ""); + +TestCase create_pw_test_case_from_config( + const Conv2dConfig& config, + vkapi::ScalarType input_dtype, + utils::StorageType fp_storage_type, + utils::GPUMemoryLayout int8_memory_layout, + const std::string& impl_selector = "", + const PointwiseTestOptions& options = {}); + +void reference_impl(TestCase& test_case); + +void scenex_direct_reference(TestCase& test_case); + +void dw_reference_impl(TestCase& test_case); + +void pw_reference_impl(TestCase& test_case); + +int64_t quantized_conv2d_flop_calculator(const TestCase& test_case); + +int64_t quantized_conv2d_dw_flop_calculator(const TestCase& test_case); + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d_dw.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d_dw.cpp similarity index 62% rename from backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d_dw.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d_dw.cpp index 6ff15f68f81..8682971620d 100644 --- a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d_dw.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d_dw.cpp @@ -4,34 +4,27 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include - -#include -#include - -#include - -#include -#include - -// #define DEBUG_MODE - -using namespace executorch::vulkan::prototyping; +#include -using namespace vkcompute; +#include +#include +#include +#include +#include -static constexpr int64_t kRefDimSizeLimit = 100; -static constexpr int64_t kRefOperationLimit = 2 * 1024 * 1024; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { // Utility function to create a test case from a Conv2dConfig for depthwise // convolution -TestCase create_test_case_from_config( +TestCase create_dw_test_case_from_config( const Conv2dConfig& config, vkapi::ScalarType input_dtype, utils::StorageType fp_storage_type, utils::GPUMemoryLayout int8_memory_layout, - const std::string& impl_selector = "") { + const std::string& impl_selector) { TestCase test_case; // Calculate output dimensions @@ -224,269 +217,38 @@ TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for quantized depthwise conv2d operation (for -// debugging) -std::vector generate_quantized_conv2d_dw_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - depthwise convolution - Conv2dConfig config = { - OutInChannels(8, 8), // channels (out, in) - equal for depthwise - InputSize2D(8, 8), // input_size (h, w) - KernelSize(3, 3), // kernel - Stride(1, 1), // stride - Padding(1, 1), // padding - Dilation(1, 1), // dilation - 8, // groups = channels.out for depthwise - }; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - - std::vector fp_storage_types = {utils::kTexture3D}; - - // Memory layouts for int8 tensors - test both optimized (4W4C) and general - // paths - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - // Generate test cases for each combination - for (const utils::StorageType fp_storage_type : fp_storage_types) { - for (const utils::GPUMemoryLayout int8_memory_layout : - int8_memory_layouts) { - config.test_case_name = - make_test_case_name(config, false, fp_storage_type, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); - - // For 4W4C layout, also test the legacy implementation - if (int8_memory_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"legacy_4w4c")); - } - } - } - - return test_cases; -} +// Custom FLOP calculator for quantized depthwise conv2d operation +int64_t quantized_conv2d_dw_flop_calculator(const TestCase& test_case) { + int kernel_idx = 9; // kernel_size is at index 9 for q8ta_q8csw_q8to -std::vector generate_quantized_conv2d_dw_narrow_workgroup_cases() { - std::vector test_cases; - std::vector configs = { - {OutInChannels(128, 128), - InputSize2D(7, 7), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 128}, - {OutInChannels(64, 64), - InputSize2D(9, 9), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 64}, - {OutInChannels(64, 64), - InputSize2D(13, 13), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 64}, - }; - - for (auto& config : configs) { - const bool is_performance = config.channels.out > kRefDimSizeLimit; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = make_test_case_name( - config, is_performance, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4C)); - } - return test_cases; -} + // Get input and weight dimensions + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); -// Generate test cases for quantized depthwise conv2d operation -std::vector generate_quantized_conv2d_dw_test_cases() { - std::vector test_cases; - if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { - return test_cases; - } + const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); - std::vector configs = { - // Depthwise convolutions: groups = channels.out, channels.in = - // channels.out - {OutInChannels(32, 32), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 32}, - {OutInChannels(64, 64), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 64}, - {OutInChannels(64, 64), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 64}, - {OutInChannels(80, 80), - InputSize2D(16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 80}, - {OutInChannels(16, 16), - InputSize2D(57, 33), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 16}, - // Different kernel sizes for depthwise - {OutInChannels(32, 32), - InputSize2D(64, 64), - KernelSize(5, 5), - Stride(1, 1), - Padding(2, 2), - Dilation(1, 1), - 32}, - {OutInChannels(96, 96), - InputSize2D(64, 64), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 96}, - // Performance cases - {OutInChannels(128, 128), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 128}, - {OutInChannels(64, 64), - InputSize2D(256, 256), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 64}, - {OutInChannels(288, 288), - InputSize2D(16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 288}, - {OutInChannels(32, 32), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(2, 2), - Dilation(1, 1), - 32}}; - - // Test with different storage types and data types - std::vector fp_storage_types = {utils::kTexture3D}; - - // Memory layouts for int8 tensors - test both optimized (4W4C) and general - // paths - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - // Generate test cases for each combination - for (auto& config : configs) { - bool is_performance = config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit || - config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - - config.op_name = "conv2d_q8ta_q8csw_q8to"; - - for (const utils::StorageType fp_storage_type : fp_storage_types) { - for (const utils::GPUMemoryLayout int8_memory_layout : - int8_memory_layouts) { - config.test_case_name = make_test_case_name( - config, is_performance, fp_storage_type, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); - - // For 4W4C layout, also test the legacy implementation - if (int8_memory_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"legacy_4w4c")); - } - } - } - } + int64_t N = input_sizes[0]; + int64_t C_out = output_sizes[1]; + int64_t K_h = kernel_sizes[0]; + int64_t K_w = kernel_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; - std::vector batch_configs = { - {OutInChannels(8, 8), - InputSize2D(8, 8), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 8, - 2}, - {OutInChannels(128, 128), - InputSize2D(64, 64), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 128, - 1}, - {OutInChannels(128, 128), - InputSize2D(64, 64), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 128, - 60}}; - - for (auto& config : batch_configs) { - const bool is_performance = config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = make_test_case_name( - config, is_performance, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4C1W)); - if (config.batch == 2) { - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4W4C)); - } - } + // Calculate FLOPs for quantized depthwise conv2d operation + // Each output element requires: + // - K_h * K_w multiply-accumulate operations (only one input channel per + // output channel) + // - Additional operations for quantization/dequantization + int64_t output_elements = N * C_out * H_out * W_out; + int64_t ops_per_output = K_h * K_w; - auto narrow_workgroup_cases = - generate_quantized_conv2d_dw_narrow_workgroup_cases(); - test_cases.insert( - test_cases.end(), - narrow_workgroup_cases.begin(), - narrow_workgroup_cases.end()); + int64_t flop = output_elements * ops_per_output; - return test_cases; + return flop; } +namespace { + // Reference implementation for activation, weight, and output quantized // depthwise conv2d void conv2d_q8ta_q8csw_q8to_dw_reference_impl(TestCase& test_case) { @@ -681,80 +443,13 @@ void conv2d_q8ta_q8csw_q8to_dw_reference_impl(TestCase& test_case) { } } -void reference_impl(TestCase& test_case) { - conv2d_q8ta_q8csw_q8to_dw_reference_impl(test_case); -} - -// Custom FLOP calculator for quantized depthwise conv2d operation -int64_t quantized_conv2d_dw_flop_calculator(const TestCase& test_case) { - int kernel_idx = 9; // kernel_size is at index 9 for q8ta_q8csw_q8to - - // Get input and weight dimensions - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - - const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); - - int64_t N = input_sizes[0]; - int64_t C_out = output_sizes[1]; - int64_t K_h = kernel_sizes[0]; - int64_t K_w = kernel_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - // Calculate FLOPs for quantized depthwise conv2d operation - // Each output element requires: - // - K_h * K_w multiply-accumulate operations (only one input channel per - // output channel) - // - Additional operations for quantization/dequantization - int64_t output_elements = N * C_out * H_out * W_out; - int64_t ops_per_output = K_h * K_w; - - int64_t flop = output_elements * ops_per_output; +} // namespace - return flop; +void dw_reference_impl(TestCase& test_case) { + conv2d_q8ta_q8csw_q8to_dw_reference_impl(test_case); } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "Quantized Depthwise Conv2d Operation with Output Quantization Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - bool narrow_workgroups_only = false; - for (int i = 1; i < argc; ++i) { - const std::string arg(argv[i]); - if (arg == "--narrow-workgroups-only") { - narrow_workgroups_only = true; - } else { - std::cerr << "Unknown argument: " << arg << std::endl; - return 2; - } - } -#ifdef DEBUG_MODE - auto test_case_generator = generate_quantized_conv2d_dw_easy_cases; -#else - auto test_case_generator = generate_quantized_conv2d_dw_test_cases; -#endif - if (narrow_workgroups_only) { - test_case_generator = generate_quantized_conv2d_dw_narrow_workgroup_cases; - } - - auto results = execute_test_cases( - test_case_generator, - quantized_conv2d_dw_flop_calculator, - "QuantizedDepthwiseInt8Conv2d", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d_pw.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d_pw.cpp similarity index 61% rename from backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d_pw.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d_pw.cpp index 8062b25d31d..244825c7525 100644 --- a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d_pw.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d_pw.cpp @@ -4,40 +4,28 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. +#include + #include +#include #include +#include +#include #include -#include -#include -#include - -#include - -#include -#include - -// #define DEBUG_MODE - -using namespace executorch::vulkan::prototyping; - -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 100; - -struct PointwiseTestOptions { - int32_t input_zero_point = 2; - bool has_bias = true; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { // Utility function to create a test case from a Conv2dConfig -static TestCase create_test_case_from_config( +TestCase create_pw_test_case_from_config( const Conv2dConfig& config, vkapi::ScalarType input_dtype, utils::StorageType fp_storage_type, utils::GPUMemoryLayout int8_memory_layout, - const std::string& impl_selector = "", - const PointwiseTestOptions& options = {}) { + const std::string& impl_selector, + const PointwiseTestOptions& options) { TestCase test_case; // Calculate output dimensions @@ -86,12 +74,7 @@ static TestCase create_test_case_from_config( input_dtype, fp_storage_type, fp_memory_layout, -#ifdef DEBUG_MODE - DataGenType::RANDOM -#else - DataGenType::RANDOM -#endif - ); + DataGenType::RANDOM); if (debugging()) { print_valuespec_data(input_tensor, "input_tensor"); @@ -233,228 +216,10 @@ static TestCase create_test_case_from_config( return test_case; } -// Generate test cases for quantized pointwise conv2d operation. When -// pw_selector is non-empty ("pw_signed", "pw_unsigned", "pw_auto"), every -// non-legacy case carries that selector so the test graph builder can force or -// verify the unsigned-dot routing instead of using the default automatic path. -static std::vector generate_quantized_conv2d_pw_test_cases( - const std::string& pw_selector = "") { - std::vector test_cases; - if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { - return test_cases; - } - - std::vector configs = { - // OC < 4 cases to test edge cases with partial output channel blocks - {OutInChannels(1, 16), - InputSize2D(8, 8), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(2, 16), - InputSize2D(8, 8), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(3, 16), - InputSize2D(8, 8), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - // Pointwise convolutions: kernel size 1x1 - {OutInChannels(32, 3), - InputSize2D(64, 64), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(32, 32), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(96, 64), - InputSize2D(16, 16), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(13, 7), - InputSize2D(57, 33), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(80, 40), - InputSize2D(64, 64), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - // Performance cases (pointwise - will use im2col) - {OutInChannels(160, 480), - InputSize2D(8, 8), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(22, 48), - InputSize2D(256, 256), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(48, 48), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(128, 128), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(64, 576), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - }; - - // Test with different storage types and memory layouts - std::vector fp_storage_types = {utils::kTexture3D}; - - // Memory layouts for int8 tensors - test both optimized (4W4C) and general - // paths - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - // Generate test cases for each combination - for (auto& config : configs) { - bool is_performance = config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit || - config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - - config.op_name = "conv2d_q8ta_q8csw_q8to"; - - for (const utils::StorageType fp_storage_type : fp_storage_types) { - for (const utils::GPUMemoryLayout int8_memory_layout : - int8_memory_layouts) { - config.test_case_name = make_test_case_name( - config, is_performance, fp_storage_type, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - pw_selector)); - - // For 4W4C layout, also test the legacy implementation - if (int8_memory_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"legacy_4w4c")); - } - } - } - } - - std::vector batch_configs = { - {OutInChannels(8, 8), - InputSize2D(8, 8), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1, - 2}, - {OutInChannels(64, 32), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1, - 1}, - {OutInChannels(64, 32), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1, - 60}}; - - for (auto& config : batch_configs) { - const bool is_performance = config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = make_test_case_name( - config, is_performance, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4C1W, - pw_selector)); - if (config.batch == 2) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - pw_selector)); - } - } - - Conv2dConfig edge_config{ - OutInChannels(13, 7), - InputSize2D(7, 5), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}; - edge_config.op_name = "conv2d_q8ta_q8csw_q8to"; - edge_config.test_case_name = make_test_case_name( - edge_config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - edge_config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - pw_selector, - {.input_zero_point = -128, .has_bias = false})); - - return test_cases; -} +namespace { // Reference implementation for activation, weight, and output quantized conv2d -static void conv2d_q8ta_q8csw_q8to_reference_impl(TestCase& test_case) { +void conv2d_q8ta_q8csw_q8to_pw_reference_impl(TestCase& test_case) { // Extract input specifications int32_t idx = 0; const ValueSpec& input_spec = test_case.inputs()[idx++]; @@ -656,93 +421,13 @@ static void conv2d_q8ta_q8csw_q8to_reference_impl(TestCase& test_case) { } } -static void reference_impl(TestCase& test_case) { - conv2d_q8ta_q8csw_q8to_reference_impl(test_case); -} - -// Custom FLOP calculator for quantized pointwise conv2d operation -static int64_t quantized_conv2d_flop_calculator(const TestCase& test_case) { - int kernel_idx = 9; // kernel_size is at index 9 for q8ta_q8csw_q8to - - // Get input and weight dimensions - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); +} // namespace - const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t C_out = output_sizes[1]; - int64_t K_h = kernel_sizes[0]; - int64_t K_w = kernel_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - // Calculate FLOPs for quantized conv2d operation - // Each output element requires: - // - C_in * K_h * K_w multiply-accumulate operations - // - Additional operations for quantization/dequantization - int64_t output_elements = N * C_out * H_out * W_out; - int64_t ops_per_output = C_in * K_h * K_w; - - int64_t flop = output_elements * (ops_per_output); - - return flop; +void pw_reference_impl(TestCase& test_case) { + conv2d_q8ta_q8csw_q8to_pw_reference_impl(test_case); } -int main(int argc, char* argv[]) { - const vkapi::Adapter& adapter = *vkcompute::api::context()->adapter_ptr(); - const bool prefers_unsigned_dot = - adapter.accelerates_unsigned_packed4x8_dot() && - !adapter.accelerates_signed_packed4x8_dot(); - VK_CHECK_COND( - can_use_unsigned_pw_dot(adapter, kMaxUnsignedDotAccumulatorBytes) == - prefers_unsigned_dot); - VK_CHECK_COND( - !can_use_unsigned_pw_dot(adapter, kMaxUnsignedDotAccumulatorBytes + 1)); - - std::string pw_impl_selector; - for (int i = 1; i < argc; ++i) { - const std::string arg(argv[i]); - if (arg == "--pw-path=signed") { - pw_impl_selector = "pw_signed"; - } else if (arg == "--pw-path=unsigned") { - pw_impl_selector = "pw_unsigned"; - } else if (arg == "--pw-path=auto") { - pw_impl_selector = "pw_auto"; - } else { - std::cerr << "Unknown argument: " << arg << std::endl; - return 2; - } - } - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(true); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "Quantized Pointwise Conv2d (1x1) Operation Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - // Execute test cases using the new framework with custom FLOP calculator - const auto test_case_generator = [pw_impl_selector]() { - return generate_quantized_conv2d_pw_test_cases(pw_impl_selector); - }; - auto results = execute_test_cases( - test_case_generator, - quantized_conv2d_flop_calculator, - "QuantizedConv2dPW", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d.cpp deleted file mode 100644 index 683dbf8506c..00000000000 --- a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_q8ta_conv2d.cpp +++ /dev/null @@ -1,1594 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include - -#include -#include - -// #define DEBUG_MODE - -using namespace executorch::vulkan::prototyping; - -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 100; -static constexpr int64_t kRefOperationLimit = 2 * 1024 * 1024; - -struct Im2colUnsignedTestOptions { - int32_t input_zero_point = 2; - int32_t output_zero_point = -1; - bool use_extreme_values = false; - bool use_accumulator_limit_values = false; - bool has_bias = true; - const char* activation = "none"; - float weight_scale = 1.0f / 256.0f; -}; - -// Utility function to create a test case from a Conv2dConfig -static TestCase create_test_case_from_config_with_layouts( - const Conv2dConfig& config, - vkapi::ScalarType input_dtype, - utils::StorageType fp_storage_type, - utils::GPUMemoryLayout input_int8_memory_layout, - utils::GPUMemoryLayout output_int8_memory_layout, - const std::string& impl_selector = "", - const Im2colUnsignedTestOptions* im2col_options = nullptr, - const float input_scale_val = 0.008123f, - const DataGenType input_data_gen = DataGenType::RANDOM) { - TestCase test_case; - - // Calculate output dimensions - int64_t H_out = config.get_output_height(); - int64_t W_out = config.get_output_width(); - - // Input tensor (float/half) - [N, C_in, H_in, W_in] - std::vector input_size = { - config.batch, - config.channels.in, - config.input_size.h, - config.input_size.w}; - - utils::GPUMemoryLayout fp_memory_layout = fp_storage_type == utils::kBuffer - ? utils::kWidthPacked - : utils::kChannelsPacked; - - // Create test case name - std::string prefix = config.test_case_name.substr(0, 4); // "ACCU" or "PERF" - std::string dtype_str = dtype_short(input_dtype); - std::string in_shape = "[" + std::to_string(config.batch) + "," + - std::to_string(config.channels.in) + "," + - std::to_string(config.input_size.h) + "," + - std::to_string(config.input_size.w) + "]"; - std::string weight_shape = "[" + std::to_string(config.channels.out) + "," + - std::to_string(config.channels.in / config.groups) + "," + - std::to_string(config.kernel.h) + "," + std::to_string(config.kernel.w) + - "]"; - std::string shape_str = in_shape + "x" + weight_shape + " s" + - std::to_string(config.stride.h) + " p" + - std::to_string(config.padding.h) + " d" + - std::to_string(config.dilation.h) + " g" + std::to_string(config.groups); - std::string storage_str = repr_str(utils::kBuffer, input_int8_memory_layout); - if (input_int8_memory_layout != output_int8_memory_layout) { - storage_str += "->" + repr_str(utils::kBuffer, output_int8_memory_layout); - } - std::string suffix = impl_selector.empty() ? "" : "[" + impl_selector + "]"; - std::string test_name = make_test_label( - prefix, dtype_str, dtype_str, shape_str, storage_str, suffix); - test_case.set_name(test_name); - - // Set the operator name for the test case - use the unified test operator - std::string operator_name = "test_etvk.test_q8ta_conv2d.default"; - test_case.set_operator_name(operator_name); - - ValueSpec input_tensor( - input_size, - input_dtype, - fp_storage_type, - fp_memory_layout, - input_data_gen); - - if (debugging()) { - print_valuespec_data(input_tensor, "input_tensor"); - } - - ValueSpec input_scale(input_scale_val); - - const int32_t input_zero_point_val = - im2col_options == nullptr ? 2 : im2col_options->input_zero_point; - ValueSpec input_zero_point(input_zero_point_val); - - if (im2col_options != nullptr && - im2col_options->use_accumulator_limit_values) { - input_tensor.ensure_data_generated(2401); - std::fill( - input_tensor.get_float_data().begin(), - input_tensor.get_float_data().end(), - (127.0f - input_zero_point_val) * input_scale_val); - } else if (im2col_options != nullptr && im2col_options->use_extreme_values) { - input_tensor.ensure_data_generated(2401); - constexpr std::array values = {-128, -1, 0, 1, 127}; - std::vector& input_data = input_tensor.get_float_data(); - for (size_t i = 0; i < input_data.size(); ++i) { - input_data.at(i) = (static_cast(values.at(i % values.size())) - - input_zero_point_val) * - input_scale_val; - } - } - - // Quantized weight tensor (int8) - [C_out, C_in_per_group * K_h * K_w] - // Memory layout: height, width, then channels - in_c is innermost (stride 1) - // in the second dimension - const int64_t in_channels_per_group = config.channels.in / config.groups; - const int64_t in_features = utils::align_up_4( - in_channels_per_group * config.kernel.h * config.kernel.w); - std::vector weight_size = {config.channels.out, in_features}; - ValueSpec quantized_weight( - weight_size, - vkapi::kChar, // int8 for quantized weights - fp_storage_type, - utils::kWidthPacked, - DataGenType::RANDINT8); - quantized_weight.set_constant(true); - - if (im2col_options != nullptr && - im2col_options->use_accumulator_limit_values) { - quantized_weight.ensure_data_generated(2402); - std::fill( - quantized_weight.get_int8_data().begin(), - quantized_weight.get_int8_data().end(), - 127); - } else if (im2col_options != nullptr && im2col_options->use_extreme_values) { - quantized_weight.ensure_data_generated(2402); - constexpr std::array values = {-128, -1, 0, 1, 127}; - std::vector& weight_data = quantized_weight.get_int8_data(); - for (size_t i = 0; i < weight_data.size(); ++i) { - weight_data.at(i) = values.at((i * 3 + 1) % values.size()); - } - } - - if (debugging()) { - print_valuespec_data(quantized_weight, "weight_tensor"); - } - - const int64_t aligned_out_channels = utils::align_up_4(config.channels.out); - - // Weight quantization scales (float/half, per-channel) - ValueSpec weight_scales( - {aligned_out_channels}, // Per output channel - input_dtype, - fp_storage_type, - utils::kWidthPacked, - DataGenType::RANDOM_SCALES); - weight_scales.set_constant(true); - if (im2col_options != nullptr) { - weight_scales.ensure_data_generated(2403); - std::fill( - weight_scales.get_float_data().begin(), - weight_scales.get_float_data().end(), - im2col_options->weight_scale); - } - - ValueSpec weight_sums( - {aligned_out_channels}, // Per output channel - vkapi::kInt, - fp_storage_type, - utils::kWidthPacked, - DataGenType::ZEROS); - weight_sums.set_constant(true); - - // Compute weight_sums data based on quantized weights - compute_weight_sums( - weight_sums, quantized_weight, config.channels.out, in_features); - - // Bias (optional, float/half) - [C_out] - ValueSpec bias( - {aligned_out_channels}, // Per output channel - input_dtype, - fp_storage_type, - utils::kWidthPacked, - DataGenType::ZEROS); - bias.set_constant(true); - if (im2col_options != nullptr && !im2col_options->has_bias) { - bias.set_none(true); - } - - // Output quantization parameters - float output_scale_val = 0.05314; - ValueSpec output_scale(output_scale_val); - - const int32_t output_zero_point_val = - im2col_options == nullptr ? -1 : im2col_options->output_zero_point; - ValueSpec output_zero_point(output_zero_point_val); - - // Stride and padding parameters - ValueSpec stride({config.stride.h, config.stride.w}); - ValueSpec padding({config.padding.h, config.padding.w}); - - // Dilation and groups parameters - ValueSpec dilation({config.dilation.h, config.dilation.w}); - ValueSpec groups(config.groups); - - // Kernel size parameters - ValueSpec kernel_size({config.kernel.h, config.kernel.w}); - - // Output tensor (float/half) - [N, C_out, H_out, W_out] - ValueSpec output( - {config.batch, config.channels.out, H_out, W_out}, - input_dtype, - fp_storage_type, - fp_memory_layout, - DataGenType::ZEROS); - - // Add all specs to test case for q8ta_q8csw_q8to operation - test_case.add_input_spec(input_tensor); - test_case.add_input_spec(input_scale); - test_case.add_input_spec(input_zero_point); - test_case.add_input_spec(quantized_weight); - test_case.add_input_spec(weight_sums); - test_case.add_input_spec(weight_scales); - test_case.add_input_spec(output_scale); - test_case.add_input_spec(output_zero_point); - test_case.add_input_spec(bias); - test_case.add_input_spec(kernel_size); - test_case.add_input_spec(stride); - test_case.add_input_spec(padding); - test_case.add_input_spec(dilation); - test_case.add_input_spec(groups); - - // Activation (none = no activation) - ValueSpec activation = ValueSpec::make_string( - im2col_options == nullptr ? "none" : im2col_options->activation); - test_case.add_input_spec(activation); - - // Add memory layout parameter for the quantized tensors - ValueSpec input_layout_int(static_cast(input_int8_memory_layout)); - test_case.add_input_spec(input_layout_int); - - ValueSpec output_layout_int(static_cast(output_int8_memory_layout)); - test_case.add_input_spec(output_layout_int); - - // Add impl_selector string - ValueSpec impl_selector_spec = ValueSpec::make_string(impl_selector); - test_case.add_input_spec(impl_selector_spec); - - test_case.add_output_spec(output); - - test_case.set_abs_tolerance( - im2col_options == nullptr ? output_scale_val + 1e-4f - : output_scale_val * 0.25f); - - // Filter out quantize/dequantize shaders from timing measurements - test_case.set_shader_filter({ - "nchw_to", - "to_nchw", - "q8ta_quantize", - "q8ta_dequantize", - }); - - return test_case; -} - -static TestCase create_test_case_from_config( - const Conv2dConfig& config, - vkapi::ScalarType input_dtype, - utils::StorageType fp_storage_type, - utils::GPUMemoryLayout int8_memory_layout, - const std::string& impl_selector = "", - const Im2colUnsignedTestOptions* im2col_options = nullptr, - const float input_scale_val = 0.008123f, - const DataGenType input_data_gen = DataGenType::RANDOM) { - return create_test_case_from_config_with_layouts( - config, - input_dtype, - fp_storage_type, - int8_memory_layout, - int8_memory_layout, - impl_selector, - im2col_options, - input_scale_val, - input_data_gen); -} - -static std::vector generate_narrow_workgroup_test_cases() { - std::vector test_cases; - std::vector configs = { - {OutInChannels(64, 32), - InputSize2D(9, 9), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(128, 32), - InputSize2D(7, 7), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - }; - - for (auto& config : configs) { - const bool is_performance = config.channels.out > kRefDimSizeLimit; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = make_test_case_name( - config, is_performance, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4C, - /*impl_selector=*/"general")); - } - return test_cases; -} - -// Generate easy test cases for quantized conv2d operation (for debugging) -std::vector generate_quantized_conv2d_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - Conv2dConfig config = { - OutInChannels(16, 8), // channels (out, in) - InputSize2D(5, 5), // input_size (h, w) - KernelSize(3, 3), // kernel - Stride(1, 1), // stride - Padding(1, 1), // padding - Dilation(1, 1), // dilation - 1, // groups - }; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - - std::vector fp_storage_types = {utils::kTexture3D}; - - // Memory layouts for int8 tensors - test both optimized (4W4C) and general - // paths - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - // Generate test cases for each combination - for (const utils::StorageType fp_storage_type : fp_storage_types) { - for (const utils::GPUMemoryLayout int8_memory_layout : - int8_memory_layouts) { - config.test_case_name = - make_test_case_name(config, false, fp_storage_type, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); - - // Test im2col implementation when input channels per group is a - // multiple of 4 - if ((config.channels.in / config.groups) % 4 == 0) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"im2col")); - } - // For 4W4C layout, also test the legacy implementation - if (int8_memory_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"legacy_4w4c")); - } - } - } - - return test_cases; -} - -static std::vector generate_im2col_unsigned_test_cases( - const std::string& impl_selector); - -static std::vector generate_streaming_im2col_test_cases() { - std::vector test_cases; - - Conv2dConfig full_fit_config = { - OutInChannels(4, 32), - InputSize2D(30, 99), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1, - 10}; - full_fit_config.op_name = "conv2d_q8ta_q8csw_q8to"; - full_fit_config.test_case_name = make_test_case_name( - full_fit_config, false, utils::kTexture3D, utils::kBuffer); - - Conv2dConfig streaming_fallback_config = full_fit_config; - streaming_fallback_config.channels.in = 64; - streaming_fallback_config.test_case_name = make_test_case_name( - streaming_fallback_config, false, utils::kTexture3D, utils::kBuffer); - - // Forced-fallback cases need no int8 dot-product support, so they run on - // all devices; the kAuto cases below stay gated. - test_cases.push_back(create_test_case_from_config( - full_fit_config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - /*impl_selector=*/"im2col_fallback", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT)); - test_cases.push_back(create_test_case_from_config( - streaming_fallback_config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - /*impl_selector=*/"im2col_fallback", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT)); - - if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { - return test_cases; - } - - for (const utils::GPUMemoryLayout layout : - {utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C}) { - test_cases.push_back(create_test_case_from_config( - full_fit_config, - vkapi::kFloat, - utils::kTexture3D, - layout, - /*impl_selector=*/"im2col_auto", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT)); - } - Conv2dConfig streaming_config = full_fit_config; - streaming_config.channels.in = 64; - streaming_config.test_case_name = make_test_case_name( - streaming_config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - streaming_config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4C1W, - /*impl_selector=*/"im2col_auto", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT)); - Conv2dConfig grouped_config = { - OutInChannels(16, 32), - InputSize2D(30, 99), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2, - 20}; - grouped_config.op_name = "conv2d_q8ta_q8csw_q8to"; - grouped_config.test_case_name = make_test_case_name( - grouped_config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - grouped_config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - /*impl_selector=*/"im2col_auto", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT)); - - Conv2dConfig output_channel_tail_config = { - OutInChannels(10, 32), - InputSize2D(30, 99), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1, - 20}; - output_channel_tail_config.op_name = "conv2d_q8ta_q8csw_q8to"; - output_channel_tail_config.test_case_name = make_test_case_name( - output_channel_tail_config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - output_channel_tail_config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - /*impl_selector=*/"im2col_auto", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT)); - return test_cases; -} - -// SceneX route tests. The kPackedInt8_4C input + kPackedInt8_4W4C output -// layout combination is only exercised here (default generators never pair -// them); run via --scenex-regular . Zero -// tolerances are exact by construction: both pipelines accumulate in int32 -// with identical requantize, so any mismatch is a real regression, not noise. -static TestCase create_scenex_test_case( - const Conv2dConfig& source_config, - const std::string& route) { - Conv2dConfig config = source_config; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = - make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); - - const std::string impl_selector = route == "auto" ? "" - : route == "direct" ? "general" - : "im2col"; - TestCase test_case = create_test_case_from_config_with_layouts( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - impl_selector, - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT); - test_case.set_abs_tolerance(0.0f); - test_case.set_rel_tolerance(0.0f); - return test_case; -} - -static TestCase generate_scenex_regular_test_case( - const std::string& route, - const int case_index) { - const std::vector configs = { - {OutInChannels(128, 64), - InputSize2D(40, 51), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1, - 60}, - {OutInChannels(256, 128), - InputSize2D(20, 26), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1, - 60}, - }; - return create_scenex_test_case(configs.at(case_index), route); -} - -static TestCase generate_scenex_grouped_test_case( - const std::string& route, - const int case_index) { - const std::vector configs = { - {OutInChannels(64, 64), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2, - 60}, - {OutInChannels(128, 128), - InputSize2D(128, 128), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 4, - 60}, - {OutInChannels(64, 64), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2, - 60}, - }; - - return create_scenex_test_case(configs.at(case_index), route); -} - -// Generate test cases for quantized conv2d operation -static std::vector generate_quantized_conv2d_test_cases() { - std::vector test_cases; - api::Context* const context = vkcompute::api::context(); - if (!context->adapter_ptr()->supports_int8_dot_product()) { - for (const std::string& impl_selector : {"im2col", "im2col_auto"}) { - std::vector im2col_cases = - generate_im2col_unsigned_test_cases(impl_selector); - for (TestCase& test_case : im2col_cases) { - test_cases.push_back(std::move(test_case)); - } - } - return test_cases; - } - - std::vector configs = { - // General 2D convolutions - {OutInChannels(32, 3), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(32, 3), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(8, 8), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(16, 32), - InputSize2D(77, 77), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - // Grouped convolutions - {OutInChannels(64, 32), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2}, - {OutInChannels(96, 96), - InputSize2D(81, 81), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 3}, - {OutInChannels(96, 96), - InputSize2D(64, 64), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 4}, - // Performance cases (3x3 convs - will use im2col) - {OutInChannels(32, 3), - InputSize2D(256, 256), - KernelSize(3, 3), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(64, 64), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - // Performance cases (grouped convs) - {OutInChannels(64, 64), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2}, - {OutInChannels(96, 96), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 3}, - {OutInChannels(128, 128), - InputSize2D(128, 128), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 4}, - // SceneX v9 grouped convolutions (large spatial) - {OutInChannels(128, 128), - InputSize2D(256, 256), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 4}, - {OutInChannels(64, 64), - InputSize2D(256, 256), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2}, - // Deep channels + small spatial (ResNet50 stage 5 bottleneck) - {OutInChannels(512, 512), - InputSize2D(7, 7), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - // Strided 1x1 shortcut (worst-case strided downsample) - {OutInChannels(2048, 1024), - InputSize2D(14, 14), - KernelSize(1, 1), - Stride(2, 2), - Padding(0, 0), - Dilation(1, 1), - 1}}; - - // Test with different storage types and memory layouts - std::vector fp_storage_types = {utils::kTexture3D}; - - // Memory layouts for int8 tensors - test both optimized (4W4C) and general - // paths - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - // Generate test cases for each combination - for (auto& config : configs) { - bool is_performance = config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit || - config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - - config.op_name = "conv2d_q8ta_q8csw_q8to"; - - for (const utils::StorageType fp_storage_type : fp_storage_types) { - for (const utils::GPUMemoryLayout int8_memory_layout : - int8_memory_layouts) { - config.test_case_name = make_test_case_name( - config, is_performance, fp_storage_type, utils::kBuffer); - - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"general")); - - // Test im2col implementation when input channels per group is a - // multiple of 4 - const int64_t in_channels_per_group = - config.channels.in / config.groups; - if (in_channels_per_group % 4 == 0) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"im2col")); - } - - // For 4W4C layout, also test the legacy implementation - if (int8_memory_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"legacy_4w4c")); - } - - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); - } - } - } - - std::vector batch_configs = { - {OutInChannels(16, 32), - InputSize2D(7, 7), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1, - 2}, - {OutInChannels(32, 3), - InputSize2D(256, 256), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1, - 1}, - {OutInChannels(32, 3), - InputSize2D(256, 256), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1, - 60}, - {OutInChannels(512, 256), - InputSize2D(10, 13), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1, - 60}}; - - for (auto& config : batch_configs) { - const bool is_performance = config.batch > kRefDimSizeLimit || - config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit || - config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = make_test_case_name( - config, is_performance, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4C1W)); - if (config.batch == 2) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4C1W, - /*impl_selector=*/"im2col")); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4W4C)); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - /*impl_selector=*/"im2col")); - } - } - - for (const std::string& impl_selector : - {"im2col", "im2col_unsigned", "im2col_auto"}) { - std::vector im2col_cases = - generate_im2col_unsigned_test_cases(impl_selector); - for (TestCase& test_case : im2col_cases) { - test_cases.push_back(std::move(test_case)); - } - } - - auto narrow_workgroup_cases = generate_narrow_workgroup_test_cases(); - test_cases.insert( - test_cases.end(), - narrow_workgroup_cases.begin(), - narrow_workgroup_cases.end()); - - auto streaming_cases = generate_streaming_im2col_test_cases(); - test_cases.insert( - test_cases.end(), streaming_cases.begin(), streaming_cases.end()); - - return test_cases; -} - -static std::vector generate_im2col_unsigned_test_cases( - const std::string& impl_selector) { - api::Context* const context = vkcompute::api::context(); - if (impl_selector == "im2col_unsigned" && - !context->adapter_ptr()->supports_int8_dot_product()) { - return {}; - } - - std::vector> configs = { - {{OutInChannels(5, 4), - InputSize2D(5, 5), - KernelSize(3, 3), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {.input_zero_point = 2, - .output_zero_point = -1, - .use_extreme_values = true, - .use_accumulator_limit_values = false, - .has_bias = true, - .activation = "none"}}, - {{OutInChannels(8, 8), - InputSize2D(5, 5), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {.input_zero_point = -7, - .output_zero_point = 3, - .use_extreme_values = true, - .use_accumulator_limit_values = false, - .has_bias = false, - .activation = "none"}}, - {{OutInChannels(12, 8), - InputSize2D(7, 7), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - {.input_zero_point = 127, - .output_zero_point = -5, - .use_extreme_values = true, - .use_accumulator_limit_values = false, - .has_bias = true, - .activation = "relu"}}, - {{OutInChannels(12, 8), - InputSize2D(9, 9), - KernelSize(3, 3), - Stride(1, 1), - Padding(2, 2), - Dilation(2, 2), - 1}, - {.input_zero_point = -128, - .output_zero_point = 5, - .use_extreme_values = true, - .use_accumulator_limit_values = false, - .has_bias = false, - .activation = "none"}}, - {{OutInChannels(8, 8), - InputSize2D(6, 7), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2}, - {.input_zero_point = 11, - .output_zero_point = -3, - .use_extreme_values = true, - .use_accumulator_limit_values = false, - .has_bias = true, - .activation = "relu"}}, - {{OutInChannels(1, 4), - InputSize2D(90, 91), - KernelSize(90, 91), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {.input_zero_point = 0, - .output_zero_point = -1, - .use_extreme_values = false, - .use_accumulator_limit_values = true, - .has_bias = true, - .activation = "none", - .weight_scale = 1.0f / 1000000.0f}}, - }; - - std::vector test_cases; - test_cases.reserve(configs.size()); - for (auto& [config, options] : configs) { - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = - make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - impl_selector, - &options)); - } - - if (impl_selector == "im2col_auto") { - Conv2dConfig config{ - OutInChannels(1, 4), - InputSize2D(91, 91), - KernelSize(91, 91), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}; - Im2colUnsignedTestOptions options; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = - make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - impl_selector, - &options)); - } - - const vkapi::Adapter& adapter = *context->adapter_ptr(); - if (impl_selector == "im2col_unsigned" || - (impl_selector == "im2col_auto" && can_use_unsigned_pw_dot(adapter, 4))) { - const int32_t buffer_output_channels = utils::safe_downcast( - static_cast(adapter.max_texture2d_dim()) * 4 + 1); - Conv2dConfig config{ - OutInChannels(buffer_output_channels, 4), - InputSize2D(1, 1), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}; - Im2colUnsignedTestOptions options; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = - make_test_case_name(config, false, utils::kBuffer, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kBuffer, - utils::kPackedInt8_4W4C, - impl_selector, - &options)); - } - - return test_cases; -} - -// Reference implementation for activation, weight, and output quantized conv2d -static void conv2d_q8ta_q8csw_q8to_reference_impl(TestCase& test_case) { - // Extract input specifications - int32_t idx = 0; - const ValueSpec& input_spec = test_case.inputs()[idx++]; - const ValueSpec& input_scale_spec = test_case.inputs()[idx++]; - const ValueSpec& input_zeros_spec = test_case.inputs()[idx++]; - const ValueSpec& weight_spec = test_case.inputs()[idx++]; - const ValueSpec& weight_sums_spec = test_case.inputs()[idx++]; - (void)weight_sums_spec; - const ValueSpec& weight_scales_spec = test_case.inputs()[idx++]; - const ValueSpec& output_scale_spec = test_case.inputs()[idx++]; - const ValueSpec& output_zeros_spec = test_case.inputs()[idx++]; - const ValueSpec& bias_spec = test_case.inputs()[idx++]; - const ValueSpec& kernel_size_spec = test_case.inputs()[idx++]; - const ValueSpec& stride_spec = test_case.inputs()[idx++]; - const ValueSpec& padding_spec = test_case.inputs()[idx++]; - const ValueSpec& dilation_spec = test_case.inputs()[idx++]; - const ValueSpec& groups_spec = test_case.inputs()[idx++]; - const ValueSpec& activation_spec = test_case.inputs()[idx++]; - const ValueSpec& layout_spec = test_case.inputs()[idx++]; - (void)layout_spec; // Not used in reference implementation - const ValueSpec& output_layout_spec = test_case.inputs()[idx++]; - (void)output_layout_spec; // Not used in reference implementation - const ValueSpec& impl_selector_spec = test_case.inputs()[idx++]; - (void)impl_selector_spec; // Not used in reference implementation - - // Extract output specification (mutable reference) - ValueSpec& output_spec = test_case.outputs()[0]; - - // Get tensor dimensions - auto input_sizes = input_spec.get_tensor_sizes(); // [N, C_in, H_in, W_in] - auto weight_sizes = - weight_spec.get_tensor_sizes(); // [C_out, C_in_per_group * K_h * K_w] - auto output_sizes = - output_spec.get_tensor_sizes(); // [N, C_out, H_out, W_out] - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t H_in = input_sizes[2]; - int64_t W_in = input_sizes[3]; - int64_t C_out = output_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - // Get kernel dimensions from kernel_size ValueSpec - auto kernel_size_data = kernel_size_spec.get_int32_data(); - int64_t K_h = kernel_size_data[0]; - int64_t K_w = kernel_size_data[1]; - - // Get stride, padding, dilation, and groups - auto stride_data = stride_spec.get_int32_data(); - auto padding_data = padding_spec.get_int32_data(); - auto dilation_data = dilation_spec.get_int32_data(); - int64_t stride_h = stride_data[0]; - int64_t stride_w = stride_data[1]; - int64_t pad_h = padding_data[0]; - int64_t pad_w = padding_data[1]; - int64_t dilation_h = dilation_data[0]; - int64_t dilation_w = dilation_data[1]; - int64_t groups = groups_spec.get_int_value(); - - const int64_t reference_operations = - N * C_out * H_out * W_out * (C_in / groups) * K_h * K_w; - const bool has_large_dimension = N > kRefDimSizeLimit || - C_in > kRefDimSizeLimit || H_in > kRefDimSizeLimit || - W_in > kRefDimSizeLimit || C_out > kRefDimSizeLimit; - if (has_large_dimension && reference_operations > kRefOperationLimit) { - throw std::invalid_argument( - "One or more dimensions exceed the allowed limit for reference implementation."); - std::cout - << "Reference implementation: computation may take some time for large tensors..." - << std::endl; - } - - if (input_spec.dtype != vkapi::kFloat) { - throw std::invalid_argument("Unsupported dtype"); - } - - // Get raw data pointers - auto& input_data = input_spec.get_float_data(); - const float input_scale = input_scale_spec.get_float_value(); - const int32_t input_zero_point = input_zeros_spec.get_int_value(); - - auto& weight_data = weight_spec.get_int8_data(); - auto& weight_scales_data = weight_scales_spec.get_float_data(); - auto& bias_data = bias_spec.get_float_data(); - - const float output_scale = output_scale_spec.get_float_value(); - const int32_t output_zero_point = output_zeros_spec.get_int_value(); - - // Calculate channels per group for grouped convolution - int64_t C_in_per_group = C_in / groups; - int64_t C_out_per_group = C_out / groups; - - // Calculate number of output elements - int64_t num_output_elements = N * C_out * H_out * W_out; - - auto& ref_data = output_spec.get_ref_float_data(); - ref_data.resize(num_output_elements); - - const int64_t in_features = utils::align_up_4(C_in_per_group * K_h * K_w); - - // Perform activation, weight, and output quantized conv2d operation - for (int64_t n = 0; n < N; ++n) { - for (int64_t out_c = 0; out_c < C_out; ++out_c) { - for (int64_t out_h = 0; out_h < H_out; ++out_h) { - for (int64_t out_w = 0; out_w < W_out; ++out_w) { - int32_t int_sum = 0; - int32_t weight_sum = 0; // Track weight sum on the fly - - // Determine which group this output channel belongs to - int64_t group_idx = out_c / C_out_per_group; - int64_t in_c_start = group_idx * C_in_per_group; - int64_t in_c_end = (group_idx + 1) * C_in_per_group; - - // Convolution operation with integer accumulation - for (int64_t in_c = in_c_start; in_c < in_c_end; ++in_c) { - for (int64_t kh = 0; kh < K_h; ++kh) { - for (int64_t kw = 0; kw < K_w; ++kw) { - // Calculate input position with dilation - int64_t in_h = out_h * stride_h - pad_h + kh * dilation_h; - int64_t in_w = out_w * stride_w - pad_w + kw * dilation_w; - - // Check bounds (zero padding) - if (in_h >= 0 && in_h < H_in && in_w >= 0 && in_w < W_in) { - // Get input value and quantize to int8 - int64_t input_idx = n * (C_in * H_in * W_in) + - in_c * (H_in * W_in) + in_h * W_in + in_w; - - float quant_input_f = - std::round(input_data[input_idx] / input_scale) + - input_zero_point; - quant_input_f = - std::min(std::max(quant_input_f, -128.0f), 127.0f); - int8_t quantized_input = static_cast(quant_input_f); - - // Get quantized weight (already int8) - // Weight layout: [C_out, C_in_per_group * K_h * K_w] - int64_t weight_idx = out_c * in_features + - (kh * (K_w * C_in_per_group) + kw * C_in_per_group + - (in_c % C_in_per_group)); - int8_t quantized_weight = weight_data[weight_idx]; - - // Integer multiplication and accumulation - int_sum += static_cast(quantized_input) * - static_cast(quantized_weight); - - // Track weight sum for this output channel on the fly - weight_sum += static_cast(quantized_weight); - } else { - // For zero padding, we still need to account for the weight - // in weight_sum when input is effectively 0 (but quantized 0 - // is input_zero_point) - int64_t weight_idx = out_c * in_features + - (kh * (K_w * C_in_per_group) + kw * C_in_per_group + - (in_c % C_in_per_group)); - int8_t quantized_weight = weight_data[weight_idx]; - - // Add contribution from zero-padded input (quantized zero = - // input_zero_point) - int_sum += static_cast(input_zero_point) * - static_cast(quantized_weight); - - // Track weight sum for this output channel on the fly - weight_sum += static_cast(quantized_weight); - } - } - } - } - - // Convert accumulated integer result to float and apply scales - // Final result = (int_sum - zero_point_correction) * input_scale * - // weight_scale + bias zero_point_correction = input_zero_point * - // sum_of_weights_for_this_output_channel - int32_t zero_point_correction = input_zero_point * weight_sum; - int32_t accum_adjusted = int_sum - zero_point_correction; - float float_result = - accum_adjusted * input_scale * weight_scales_data[out_c]; - - if (!bias_spec.is_none()) { - float_result += bias_data[out_c]; - } - if (activation_spec.get_string_value() == "relu") { - float_result = std::max(float_result, 0.0f); - } - - // Quantize the output to int8 - float quant_output_f = - std::round(float_result / output_scale) + output_zero_point; - quant_output_f = std::min(std::max(quant_output_f, -128.0f), 127.0f); - int8_t quantized_output = static_cast(quant_output_f); - - // Dequantize back to float - float dequant_output = - (static_cast(quantized_output) - output_zero_point) * - output_scale; - - int64_t output_idx = n * (C_out * H_out * W_out) + - out_c * (H_out * W_out) + out_h * W_out + out_w; - ref_data[output_idx] = dequant_output; - } - } - } - } -} - -static void reference_impl(TestCase& test_case) { - conv2d_q8ta_q8csw_q8to_reference_impl(test_case); -} - -// The impl selector holds one of "", "general", or "im2col"; activation and -// other string inputs use disjoint values. Overwrite it by value so a -// reordered input list fails loudly instead of mutating the wrong spec. -// Note: for route == "direct" the measured run already forces "general", so -// this reference re-executes the identical implementation and only checks -// determinism; genuine cross-implementation correctness comes from the -// auto/im2col legs. -static void scenex_direct_reference(TestCase& test_case) { - TestCase direct_case = test_case; - bool found_selector = false; - for (auto it = direct_case.inputs().rbegin(); - it != direct_case.inputs().rend(); - ++it) { - if (it->is_string() && - (it->get_string_value().empty() || - it->get_string_value() == "general" || - it->get_string_value() == "im2col")) { - it->string_data = "general"; - found_selector = true; - break; - } - } - if (!found_selector) { - throw std::runtime_error("scenex reference: impl selector input not found"); - } - execute_test_case( - direct_case, - /*warmup_runs=*/1, - /*benchmark_runs=*/1, - /*chained_dispatches=*/1, - /*write_outputs=*/true); - test_case.outputs().at(0).get_ref_float_data() = - direct_case.outputs().at(0).get_float_data(); -} - -// Custom FLOP calculator for quantized conv2d operation -static int64_t quantized_conv2d_flop_calculator(const TestCase& test_case) { - int kernel_idx = 9; // kernel_size is at index 9 for q8ta_q8csw_q8to - - // Get input and weight dimensions - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - - const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t C_out = output_sizes[1]; - int64_t K_h = kernel_sizes[0]; - int64_t K_w = kernel_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - // Calculate FLOPs for quantized conv2d operation - // Each output element requires: - // - C_in * K_h * K_w multiply-accumulate operations - // - Additional operations for quantization/dequantization - int64_t output_elements = N * C_out * H_out * W_out; - int64_t ops_per_output = C_in * K_h * K_w; - - int64_t flop = output_elements * (ops_per_output); - - return flop; -} - -static void execute_streaming_dynamic_shrink_test() { - Conv2dConfig config = { - OutInChannels(4, 64), - InputSize2D(30, 99), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1, - 10}; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = - make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); - TestCase test_case = create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - /*impl_selector=*/"im2col_auto", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT); - for (ValueSpec& input : test_case.inputs()) { - input.ensure_data_generated(/*seed=*/0); - } - - BenchmarkGraph benchmark_graph = setup_compute_graph( - test_case, test_case.operator_name(), /*op_invocations_per_execute=*/1); - ComputeGraph& graph = *benchmark_graph.graph; - graph.prepare(); - graph.prepack(); - - const std::vector upper_input_sizes = test_case.inputs().at(0).sizes; - const std::vector shrunk_input_sizes = {1, 64, 20, 51}; - const std::vector shrunk_output_sizes = {1, 4, 20, 51}; - TestCase shrunk_case = test_case; - shrunk_case.inputs().at(0).sizes = shrunk_input_sizes; - shrunk_case.inputs().at(0).resize_data(1 * 64 * 20 * 51); - shrunk_case.outputs().at(0).sizes = shrunk_output_sizes; - shrunk_case.outputs().at(0).resize_data(1 * 4 * 20 * 51); - reference_impl(shrunk_case); - - constexpr int kRepetitions = 4; - for (int repetition = 0; repetition < kRepetitions; ++repetition) { - graph.resize_input(0, upper_input_sizes); - graph.propagate_resize(); - graph.maybe_cast_and_copy_into_staging( - graph.inputs().at(0).staging, - test_case.inputs().at(0).get_data_ptr(), - test_case.inputs().at(0).numel(), - vkapi::kFloat); - graph.execute(); - - graph.resize_input(0, shrunk_input_sizes); - graph.propagate_resize(); - if (graph.sizes_of(graph.outputs().at(0).value) != shrunk_output_sizes) { - throw std::runtime_error("streaming im2col output did not shrink"); - } - graph.maybe_cast_and_copy_into_staging( - graph.inputs().at(0).staging, - shrunk_case.inputs().at(0).get_data_ptr(), - shrunk_case.inputs().at(0).numel(), - vkapi::kFloat); - graph.execute(); - graph.maybe_cast_and_copy_from_staging( - graph.outputs().at(0).staging, - shrunk_case.outputs().at(0).get_mutable_data_ptr(), - shrunk_case.outputs().at(0).numel(), - vkapi::kFloat); - if (!shrunk_case.outputs().at(0).validate_against_reference( - shrunk_case.get_abs_tolerance(), shrunk_case.get_rel_tolerance())) { - throw std::runtime_error("streaming im2col shrink output was stale"); - } - } - - const Q8taConv2dStreamPlan upper_plan = make_q8ta_conv2d_stream_plan( - /*batch=*/10, - /*flattened_kernel_size=*/576, - /*out_height=*/30, - /*out_width=*/99, - kQ8taConv2dIm2ColScratchBudgetBytes); - if (!upper_plan.feasible || upper_plan.num_tiles != 2 || - upper_plan.rows_per_tile <= 20) { - throw std::runtime_error("dynamic shrink test did not create dead tiles"); - } - const int64_t resized_scratch_bytes = - 576 * upper_plan.rows_per_tile * utils::align_up_4(51); - if (resized_scratch_bytes > kQ8taConv2dIm2ColScratchBudgetBytes) { - throw std::runtime_error("streaming im2col scratch exceeded its cap"); - } - std::cout << "Streaming im2col dynamic shrink PASSED" << std::endl; -} - -// Single usage string for the scenex route-test modes; argument errors -// return 2 like the other CLI errors in main. -static int print_scenex_usage(const char* mode, const char* cases) { - std::cerr << "Usage: " << mode << " <" << cases << ">" - << std::endl; - return 2; -} - -int main(int argc, char* argv[]) { - const vkapi::Adapter& adapter = *vkcompute::api::context()->adapter_ptr(); - const bool prefers_unsigned_dot = - adapter.accelerates_unsigned_packed4x8_dot() && - !adapter.accelerates_signed_packed4x8_dot(); - VK_CHECK_COND( - can_use_unsigned_pw_dot(adapter, kMaxUnsignedDotAccumulatorBytes) == - prefers_unsigned_dot); - VK_CHECK_COND( - !can_use_unsigned_pw_dot(adapter, kMaxUnsignedDotAccumulatorBytes + 1)); - - std::string im2col_impl_selector; - bool narrow_workgroups_only = false; - bool streaming_im2col_only = false; - bool streaming_dynamic_shrink_only = false; - const bool scenex_regular = - argc == 4 && std::string(argv[1]) == "--scenex-regular"; - const bool scenex_grouped = - argc == 4 && std::string(argv[1]) == "--scenex-grouped"; - if (argc >= 2 && std::string(argv[1]) == "--scenex-regular" && - !scenex_regular) { - return print_scenex_usage("--scenex-regular", "0|1"); - } - if (argc >= 2 && std::string(argv[1]) == "--scenex-grouped" && - !scenex_grouped) { - return print_scenex_usage("--scenex-grouped", "0|1|2"); - } - if (!scenex_regular && !scenex_grouped) { - for (int i = 1; i < argc; ++i) { - const std::string arg(argv[i]); - if (arg == "--im2col-path=signed") { - im2col_impl_selector = "im2col"; - } else if (arg == "--im2col-path=unsigned") { - im2col_impl_selector = "im2col_unsigned"; - } else if (arg == "--im2col-path=auto") { - im2col_impl_selector = "im2col_auto"; - } else if (arg == "--narrow-workgroups-only") { - narrow_workgroups_only = true; - } else if (arg == "--streaming-im2col-only") { - streaming_im2col_only = true; - } else if (arg == "--streaming-dynamic-shrink-only") { - streaming_dynamic_shrink_only = true; - } else { - std::cerr << "Unknown argument: " << arg << std::endl; - return 2; - } - } - } - const int selected_modes = static_cast(!im2col_impl_selector.empty()) + - static_cast(narrow_workgroups_only) + - static_cast(streaming_im2col_only) + - static_cast(streaming_dynamic_shrink_only) + - static_cast(scenex_regular) + static_cast(scenex_grouped); - if (selected_modes > 1) { - std::cerr << "Test mode selectors are mutually exclusive" << std::endl; - return 2; - } - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(true); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "Quantized Conv2d Operation with Output Quantization Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - int warmup_runs = 1; - int benchmark_runs = 1; - - if (streaming_dynamic_shrink_only) { - execute_streaming_dynamic_shrink_test(); - return 0; - } -#ifdef DEBUG_MODE - std::function()> test_case_generator = - generate_quantized_conv2d_easy_cases; -#else - std::function()> test_case_generator = - [im2col_impl_selector]() { - return im2col_impl_selector.empty() - ? generate_quantized_conv2d_test_cases() - : generate_im2col_unsigned_test_cases(im2col_impl_selector); - }; -#endif - if (narrow_workgroups_only) { - test_case_generator = generate_narrow_workgroup_test_cases; - } else if (streaming_im2col_only) { - test_case_generator = generate_streaming_im2col_test_cases; -#ifndef DEBUG_MODE - } else if (selected_modes == 0) { - // The default run also covers the unified tile path: multi-tile - // dispatches, grouped/streaming shapes, and the fallback kernel. - test_case_generator = [base_generator = test_case_generator]() { - auto cases = base_generator(); - const auto streaming_cases = generate_streaming_im2col_test_cases(); - cases.insert(cases.end(), streaming_cases.begin(), streaming_cases.end()); - return cases; - }; -#endif - } else if (scenex_regular) { - const std::string route = argv[2]; - const std::string case_arg = argv[3]; - if ((route != "auto" && route != "direct" && route != "im2col") || - (case_arg != "0" && case_arg != "1")) { - return print_scenex_usage("--scenex-regular", "0|1"); - } - const int case_index = case_arg == "0" ? 0 : 1; - test_case_generator = [route, case_index]() { - return std::vector{ - generate_scenex_regular_test_case(route, case_index)}; - }; - ref_fn = scenex_direct_reference; - warmup_runs = 3; - benchmark_runs = 10; - } else if (scenex_grouped) { - const std::string route = argv[2]; - const std::string case_arg = argv[3]; - if ((route != "auto" && route != "direct" && route != "im2col") || - (case_arg != "0" && case_arg != "1" && case_arg != "2")) { - return print_scenex_usage("--scenex-grouped", "0|1|2"); - } - const int case_index = case_arg == "0" ? 0 : case_arg == "1" ? 1 : 2; - test_case_generator = [route, case_index]() { - return std::vector{ - generate_scenex_grouped_test_case(route, case_index)}; - }; - ref_fn = scenex_direct_reference; - warmup_runs = 3; - benchmark_runs = 10; - } - - auto results = execute_test_cases( - test_case_generator, - quantized_conv2d_flop_calculator, - "QuantizedConv2dQ8ToQ8To", - warmup_runs, - benchmark_runs, - ref_fn); - -#ifndef DEBUG_MODE - if (selected_modes == 0) { - execute_streaming_dynamic_shrink_test(); - } -#endif - - return 0; -} diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/conv2d_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/conv2d_sets.cpp new file mode 100644 index 00000000000..cdabdc58969 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/conv2d_sets.cpp @@ -0,0 +1,319 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +std::vector generate_config_test_cases( + const std::vector& configs, + const std::vector& batch_configs, + const std::vector& fp_storage_types, + const std::vector& int8_memory_layouts) { + std::vector test_cases; + if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { + return test_cases; + } + + // Generate test cases for each combination + for (auto config : configs) { + bool is_performance = config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit || + config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + + config.op_name = "conv2d_q8ta_q8csw_q8to"; + + for (const utils::StorageType fp_storage_type : fp_storage_types) { + for (const utils::GPUMemoryLayout int8_memory_layout : + int8_memory_layouts) { + config.test_case_name = make_test_case_name( + config, is_performance, fp_storage_type, utils::kBuffer); + + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"general")); + + // Test im2col implementation when input channels per group is a + // multiple of 4 + const int64_t in_channels_per_group = + config.channels.in / config.groups; + if (in_channels_per_group % 4 == 0) { + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"im2col")); + } + + // For 4W4C layout, also test the legacy implementation + if (int8_memory_layout == utils::kPackedInt8_4W4C) { + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"legacy_4w4c")); + } + + test_cases.push_back(create_test_case_from_config( + config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); + } + } + } + + for (auto config : batch_configs) { + const bool is_performance = config.batch > kRefDimSizeLimit || + config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit || + config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = make_test_case_name( + config, is_performance, utils::kTexture3D, utils::kBuffer); + test_cases.push_back(create_test_case_from_config( + config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4C1W)); + if (config.batch == 2) { + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4C1W, + /*impl_selector=*/"im2col")); + test_cases.push_back(create_test_case_from_config( + config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4W4C)); + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4W4C, + /*impl_selector=*/"im2col")); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "correctness") { + const std::vector configs = { + // General 2D convolutions + {OutInChannels(32, 3), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(32, 3), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(8, 8), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(16, 32), + InputSize2D(77, 77), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + // Grouped convolutions + {OutInChannels(64, 32), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2}, + {OutInChannels(96, 96), + InputSize2D(81, 81), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 3}, + {OutInChannels(96, 96), + InputSize2D(64, 64), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 4}, + }; + const std::vector batch_configs = { + {OutInChannels(16, 32), + InputSize2D(7, 7), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 2}, + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + return { + generate_config_test_cases( + configs, batch_configs, fp_storage_types, int8_memory_layouts), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "performance") { + const std::vector configs = { + // Performance cases (3x3 convs - will use im2col) + {OutInChannels(32, 3), + InputSize2D(256, 256), + KernelSize(3, 3), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(64, 64), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + // Performance cases (grouped convs) + {OutInChannels(64, 64), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2}, + {OutInChannels(96, 96), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 3}, + {OutInChannels(128, 128), + InputSize2D(128, 128), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 4}, + // SceneX v9 grouped convolutions (large spatial) + {OutInChannels(128, 128), + InputSize2D(256, 256), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 4}, + {OutInChannels(64, 64), + InputSize2D(256, 256), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2}, + // Deep channels + small spatial (ResNet50 stage 5 bottleneck) + {OutInChannels(512, 512), + InputSize2D(7, 7), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + // Strided 1x1 shortcut (worst-case strided downsample) + {OutInChannels(2048, 1024), + InputSize2D(14, 14), + KernelSize(1, 1), + Stride(2, 2), + Padding(0, 0), + Dilation(1, 1), + 1}, + }; + const std::vector batch_configs = { + {OutInChannels(32, 3), + InputSize2D(256, 256), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1, + 1}, + {OutInChannels(32, 3), + InputSize2D(256, 256), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1, + 60}, + {OutInChannels(512, 256), + InputSize2D(10, 13), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1, + 60}, + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + return { + generate_config_test_cases( + configs, batch_configs, fp_storage_types, int8_memory_layouts), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/debug_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/debug_set.cpp new file mode 100644 index 00000000000..255f92add31 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/debug_set.cpp @@ -0,0 +1,88 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +// Generate easy test cases for quantized conv2d operation (for debugging) +std::vector generate_quantized_conv2d_easy_cases( + const Conv2dConfig& source_config, + const std::vector& fp_storage_types, + const std::vector& int8_memory_layouts) { + std::vector test_cases; + Conv2dConfig config = source_config; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + + // Generate test cases for each combination + for (const utils::StorageType fp_storage_type : fp_storage_types) { + for (const utils::GPUMemoryLayout int8_memory_layout : + int8_memory_layouts) { + config.test_case_name = + make_test_case_name(config, false, fp_storage_type, utils::kBuffer); + test_cases.push_back(create_test_case_from_config( + config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); + + // Test im2col implementation when input channels per group is a + // multiple of 4 + if ((config.channels.in / config.groups) % 4 == 0) { + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"im2col")); + } + // For 4W4C layout, also test the legacy implementation + if (int8_memory_layout == utils::kPackedInt8_4W4C) { + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"legacy_4w4c")); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "debug") { + // Single simple configuration for debugging + const Conv2dConfig config = { + OutInChannels(16, 8), // channels (out, in) + InputSize2D(5, 5), // input_size (h, w) + KernelSize(3, 3), // kernel + Stride(1, 1), // stride + Padding(1, 1), // padding + Dilation(1, 1), // dilation + 1, // groups + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + return { + generate_quantized_conv2d_easy_cases( + config, fp_storage_types, int8_memory_layouts), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_debug_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_debug_set.cpp new file mode 100644 index 00000000000..b5395dda31c --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_debug_set.cpp @@ -0,0 +1,79 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +// Generate easy test cases for quantized depthwise conv2d operation (for +// debugging) +std::vector generate_quantized_conv2d_dw_easy_cases( + const Conv2dConfig& source_config, + const std::vector& fp_storage_types, + const std::vector& int8_memory_layouts) { + std::vector test_cases; + Conv2dConfig config = source_config; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + + // Generate test cases for each combination + for (const utils::StorageType fp_storage_type : fp_storage_types) { + for (const utils::GPUMemoryLayout int8_memory_layout : + int8_memory_layouts) { + config.test_case_name = + make_test_case_name(config, false, fp_storage_type, utils::kBuffer); + test_cases.push_back(create_dw_test_case_from_config( + config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); + + // For 4W4C layout, also test the legacy implementation + if (int8_memory_layout == utils::kPackedInt8_4W4C) { + test_cases.push_back(create_dw_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"legacy_4w4c")); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "dw_debug") { + // Single simple configuration for debugging - depthwise convolution + const Conv2dConfig config = { + OutInChannels(8, 8), // channels (out, in) - equal for depthwise + InputSize2D(8, 8), // input_size (h, w) + KernelSize(3, 3), // kernel + Stride(1, 1), // stride + Padding(1, 1), // padding + Dilation(1, 1), // dilation + 8, // groups = channels.out for depthwise + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + return { + generate_quantized_conv2d_dw_easy_cases( + config, fp_storage_types, int8_memory_layouts), + dw_reference_impl, + quantized_conv2d_dw_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_narrow_workgroup_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_narrow_workgroup_set.cpp new file mode 100644 index 00000000000..49ba8573915 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_narrow_workgroup_set.cpp @@ -0,0 +1,68 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +std::vector generate_quantized_conv2d_dw_narrow_workgroup_cases( + const std::vector& configs) { + std::vector test_cases; + for (Conv2dConfig config : configs) { + const bool is_performance = config.channels.out > kRefDimSizeLimit; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = make_test_case_name( + config, is_performance, utils::kTexture3D, utils::kBuffer); + test_cases.push_back(create_dw_test_case_from_config( + config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4C)); + } + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "dw_narrow_workgroup") { + const std::vector configs = { + {OutInChannels(128, 128), + InputSize2D(7, 7), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 128}, + {OutInChannels(64, 64), + InputSize2D(9, 9), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 64}, + {OutInChannels(64, 64), + InputSize2D(13, 13), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 64}, + }; + return { + generate_quantized_conv2d_dw_narrow_workgroup_cases(configs), + dw_reference_impl, + quantized_conv2d_dw_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_sets.cpp new file mode 100644 index 00000000000..37a9d4e0cf6 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_sets.cpp @@ -0,0 +1,216 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +std::vector generate_dw_test_cases( + const std::vector& configs, + const std::vector& batch_configs, + const std::vector& fp_storage_types, + const std::vector& int8_memory_layouts) { + std::vector test_cases; + if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { + return test_cases; + } + + // Generate test cases for each combination + for (auto config : configs) { + bool is_performance = config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit || + config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + + config.op_name = "conv2d_q8ta_q8csw_q8to"; + + for (const utils::StorageType fp_storage_type : fp_storage_types) { + for (const utils::GPUMemoryLayout int8_memory_layout : + int8_memory_layouts) { + config.test_case_name = make_test_case_name( + config, is_performance, fp_storage_type, utils::kBuffer); + test_cases.push_back(create_dw_test_case_from_config( + config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); + + // For 4W4C layout, also test the legacy implementation + if (int8_memory_layout == utils::kPackedInt8_4W4C) { + test_cases.push_back(create_dw_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"legacy_4w4c")); + } + } + } + } + + for (auto config : batch_configs) { + const bool is_performance = config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = make_test_case_name( + config, is_performance, utils::kTexture3D, utils::kBuffer); + test_cases.push_back(create_dw_test_case_from_config( + config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4C1W)); + if (config.batch == 2) { + test_cases.push_back(create_dw_test_case_from_config( + config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4W4C)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "dw_correctness") { + const std::vector configs = { + // Depthwise convolutions: groups = channels.out, channels.in = + // channels.out + {OutInChannels(32, 32), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 32}, + {OutInChannels(64, 64), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 64}, + {OutInChannels(64, 64), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 64}, + {OutInChannels(80, 80), + InputSize2D(16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 80}, + {OutInChannels(16, 16), + InputSize2D(57, 33), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 16}, + // Different kernel sizes for depthwise + {OutInChannels(32, 32), + InputSize2D(64, 64), + KernelSize(5, 5), + Stride(1, 1), + Padding(2, 2), + Dilation(1, 1), + 32}, + {OutInChannels(96, 96), + InputSize2D(64, 64), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 96}, + }; + const std::vector batch_configs = { + {OutInChannels(8, 8), + InputSize2D(8, 8), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 8, + 2}, + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + return { + generate_dw_test_cases( + configs, batch_configs, fp_storage_types, int8_memory_layouts), + dw_reference_impl, + quantized_conv2d_dw_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "dw_performance") { + const std::vector configs = { + // Performance cases + {OutInChannels(128, 128), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 128}, + {OutInChannels(64, 64), + InputSize2D(256, 256), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 64}, + {OutInChannels(288, 288), + InputSize2D(16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 288}, + {OutInChannels(32, 32), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(2, 2), + Dilation(1, 1), + 32}, + }; + const std::vector batch_configs = { + {OutInChannels(128, 128), + InputSize2D(64, 64), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 128, + 1}, + {OutInChannels(128, 128), + InputSize2D(64, 64), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 128, + 60}, + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + return { + generate_dw_test_cases( + configs, batch_configs, fp_storage_types, int8_memory_layouts), + dw_reference_impl, + quantized_conv2d_dw_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/im2col_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/im2col_set.cpp new file mode 100644 index 00000000000..df7ccce3f52 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/im2col_set.cpp @@ -0,0 +1,206 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +TestCase create_im2col_test_case( + const Conv2dConfig& source_config, + utils::StorageType fp_storage_type, + const std::string& impl_selector, + const Im2colUnsignedTestOptions& options) { + Conv2dConfig config = source_config; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = + make_test_case_name(config, false, fp_storage_type, utils::kBuffer); + return create_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + utils::kPackedInt8_4W4C, + impl_selector, + &options); +} + +std::vector generate_im2col_test_cases( + const std::vector>& + configs, + const std::vector& impl_selectors, + const std::vector& auto_configs, + const std::vector& buffer_output_configs) { + const vkapi::Adapter& adapter = *vkcompute::api::context()->adapter_ptr(); + std::vector test_cases; + for (const std::string& impl_selector : impl_selectors) { + if (impl_selector == "im2col_unsigned" && + !adapter.supports_int8_dot_product()) { + continue; + } + + for (const auto& [config, options] : configs) { + test_cases.push_back(create_im2col_test_case( + config, utils::kTexture3D, impl_selector, options)); + } + + if (impl_selector == "im2col_auto") { + for (const Conv2dConfig& config : auto_configs) { + test_cases.push_back(create_im2col_test_case( + config, + utils::kTexture3D, + impl_selector, + Im2colUnsignedTestOptions{})); + } + } + + if (impl_selector == "im2col_unsigned" || + (impl_selector == "im2col_auto" && + can_use_unsigned_pw_dot(adapter, 4))) { + for (Conv2dConfig config : buffer_output_configs) { + config.channels.out = utils::safe_downcast( + static_cast(adapter.max_texture2d_dim()) * 4 + + config.channels.out); + test_cases.push_back(create_im2col_test_case( + config, + utils::kBuffer, + impl_selector, + Im2colUnsignedTestOptions{})); + } + } + } + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "im2col") { + const std::vector> + configs = { + {{OutInChannels(5, 4), + InputSize2D(5, 5), + KernelSize(3, 3), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {.input_zero_point = 2, + .output_zero_point = -1, + .use_extreme_values = true, + .use_accumulator_limit_values = false, + .has_bias = true, + .activation = "none"}}, + {{OutInChannels(8, 8), + InputSize2D(5, 5), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {.input_zero_point = -7, + .output_zero_point = 3, + .use_extreme_values = true, + .use_accumulator_limit_values = false, + .has_bias = false, + .activation = "none"}}, + {{OutInChannels(12, 8), + InputSize2D(7, 7), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + {.input_zero_point = 127, + .output_zero_point = -5, + .use_extreme_values = true, + .use_accumulator_limit_values = false, + .has_bias = true, + .activation = "relu"}}, + {{OutInChannels(12, 8), + InputSize2D(9, 9), + KernelSize(3, 3), + Stride(1, 1), + Padding(2, 2), + Dilation(2, 2), + 1}, + {.input_zero_point = -128, + .output_zero_point = 5, + .use_extreme_values = true, + .use_accumulator_limit_values = false, + .has_bias = false, + .activation = "none"}}, + {{OutInChannels(8, 8), + InputSize2D(6, 7), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2}, + {.input_zero_point = 11, + .output_zero_point = -3, + .use_extreme_values = true, + .use_accumulator_limit_values = false, + .has_bias = true, + .activation = "relu"}}, + {{OutInChannels(1, 4), + InputSize2D(90, 91), + KernelSize(90, 91), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {.input_zero_point = 0, + .output_zero_point = -1, + .use_extreme_values = false, + .use_accumulator_limit_values = true, + .has_bias = true, + .activation = "none", + .weight_scale = 1.0f / 1000000.0f}}, + }; + const std::vector impl_selectors = { + "im2col", "im2col_unsigned", "im2col_auto"}; + const std::vector auto_configs = { + {OutInChannels(1, 4), + InputSize2D(91, 91), + KernelSize(91, 91), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + }; + // Output channels are offset by max_texture2d_dim * 4 so that the output + // exceeds the texture limit and uses buffer storage. + const std::vector buffer_output_configs = { + {OutInChannels(1, 4), + InputSize2D(1, 1), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + }; + return { + generate_im2col_test_cases( + configs, impl_selectors, auto_configs, buffer_output_configs), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/narrow_workgroup_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/narrow_workgroup_set.cpp new file mode 100644 index 00000000000..896b9ff66e5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/narrow_workgroup_set.cpp @@ -0,0 +1,65 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +std::vector generate_narrow_workgroup_test_cases( + const std::vector& configs) { + std::vector test_cases; + for (Conv2dConfig config : configs) { + const bool is_performance = config.channels.out > kRefDimSizeLimit; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = make_test_case_name( + config, is_performance, utils::kTexture3D, utils::kBuffer); + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4C, + /*impl_selector=*/"general")); + } + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "narrow_workgroup") { + const std::vector configs = { + {OutInChannels(64, 32), + InputSize2D(9, 9), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(128, 32), + InputSize2D(7, 7), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + }; + return { + generate_narrow_workgroup_test_cases(configs), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/pw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/pw_sets.cpp new file mode 100644 index 00000000000..046e8f6d271 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/pw_sets.cpp @@ -0,0 +1,294 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +// Generate test cases for quantized pointwise conv2d operation. A non-empty pw +// selector lets the test graph builder force or verify the unsigned-dot +// routing instead of the default automatic path. +std::vector generate_quantized_conv2d_pw_test_cases( + const std::vector& configs, + const std::vector& batch_configs, + const std::vector>& + edge_configs, + const std::vector& fp_storage_types, + const std::vector& int8_memory_layouts, + const std::vector& pw_selectors) { + std::vector test_cases; + if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { + return test_cases; + } + + // Generate test cases for each combination + for (Conv2dConfig config : configs) { + bool is_performance = config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit || + config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + + config.op_name = "conv2d_q8ta_q8csw_q8to"; + + for (const utils::StorageType fp_storage_type : fp_storage_types) { + for (const utils::GPUMemoryLayout int8_memory_layout : + int8_memory_layouts) { + config.test_case_name = make_test_case_name( + config, is_performance, fp_storage_type, utils::kBuffer); + for (const std::string& pw_selector : pw_selectors) { + test_cases.push_back(create_pw_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + pw_selector)); + } + + // For 4W4C layout, also test the legacy implementation + if (int8_memory_layout == utils::kPackedInt8_4W4C) { + test_cases.push_back(create_pw_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"legacy_4w4c")); + } + } + } + } + + for (Conv2dConfig config : batch_configs) { + const bool is_performance = config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = make_test_case_name( + config, is_performance, utils::kTexture3D, utils::kBuffer); + for (const std::string& pw_selector : pw_selectors) { + test_cases.push_back(create_pw_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4C1W, + pw_selector)); + if (config.batch == 2) { + test_cases.push_back(create_pw_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4W4C, + pw_selector)); + } + } + } + + for (const auto& [edge_config, options] : edge_configs) { + Conv2dConfig config = edge_config; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = + make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); + for (const std::string& pw_selector : pw_selectors) { + test_cases.push_back(create_pw_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4W4C, + pw_selector, + options)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "pw_correctness") { + const std::vector configs = { + // OC < 4 cases to test edge cases with partial output channel blocks + {OutInChannels(1, 16), + InputSize2D(8, 8), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(2, 16), + InputSize2D(8, 8), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(3, 16), + InputSize2D(8, 8), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + // Pointwise convolutions: kernel size 1x1 + {OutInChannels(32, 3), + InputSize2D(64, 64), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(32, 32), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(96, 64), + InputSize2D(16, 16), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(13, 7), + InputSize2D(57, 33), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(80, 40), + InputSize2D(64, 64), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + }; + const std::vector batch_configs = { + {OutInChannels(8, 8), + InputSize2D(8, 8), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1, + 2}, + }; + const std::vector> + edge_configs = { + {{OutInChannels(13, 7), + InputSize2D(7, 5), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {.input_zero_point = -128, .has_bias = false}}, + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + const std::vector pw_selectors = { + "", "pw_signed", "pw_unsigned", "pw_auto"}; + return { + generate_quantized_conv2d_pw_test_cases( + configs, + batch_configs, + edge_configs, + fp_storage_types, + int8_memory_layouts, + pw_selectors), + pw_reference_impl, + quantized_conv2d_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "pw_performance") { + const std::vector configs = { + // Performance cases (pointwise - will use im2col) + {OutInChannels(160, 480), + InputSize2D(8, 8), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(22, 48), + InputSize2D(256, 256), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(48, 48), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(128, 128), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(64, 576), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + }; + const std::vector batch_configs = { + {OutInChannels(64, 32), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1, + 1}, + {OutInChannels(64, 32), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1, + 60}, + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + const std::vector pw_selectors = { + "", "pw_signed", "pw_unsigned", "pw_auto"}; + return { + generate_quantized_conv2d_pw_test_cases( + configs, + batch_configs, + /*edge_configs=*/{}, + fp_storage_types, + int8_memory_layouts, + pw_selectors), + pw_reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/scenex_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/scenex_sets.cpp new file mode 100644 index 00000000000..709b6f40c0e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/scenex_sets.cpp @@ -0,0 +1,101 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +std::vector generate_scenex_test_cases( + const std::vector& configs, + const std::vector& routes) { + std::vector test_cases; + for (const std::string& route : routes) { + for (const Conv2dConfig& config : configs) { + test_cases.push_back(create_scenex_test_case(config, route)); + } + } + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "scenex_regular") { + const std::vector configs = { + {OutInChannels(128, 64), + InputSize2D(40, 51), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1, + 60}, + {OutInChannels(256, 128), + InputSize2D(20, 26), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1, + 60}, + }; + const std::vector routes = {"auto", "direct", "im2col"}; + return { + generate_scenex_test_cases(configs, routes), + scenex_direct_reference, + quantized_conv2d_flop_calculator, + /*warmup_runs=*/3, + /*benchmark_runs=*/10}; +} + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "scenex_grouped") { + const std::vector configs = { + {OutInChannels(64, 64), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2, + 60}, + {OutInChannels(128, 128), + InputSize2D(128, 128), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 4, + 60}, + {OutInChannels(64, 64), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2, + 60}, + }; + const std::vector routes = {"auto", "direct", "im2col"}; + return { + generate_scenex_test_cases(configs, routes), + scenex_direct_reference, + quantized_conv2d_flop_calculator, + /*warmup_runs=*/3, + /*benchmark_runs=*/10}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_dynamic_shrink_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_dynamic_shrink_set.cpp new file mode 100644 index 00000000000..c9354b017a6 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_dynamic_shrink_set.cpp @@ -0,0 +1,128 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +void execute_streaming_dynamic_shrink_test() { + Conv2dConfig config = { + OutInChannels(4, 64), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 10}; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = + make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); + TestCase test_case = create_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4W4C, + /*impl_selector=*/"im2col_auto", + /*im2col_options=*/nullptr, + /*input_scale_val=*/1.0f, + /*input_data_gen=*/DataGenType::RANDINT); + for (ValueSpec& input : test_case.inputs()) { + input.ensure_data_generated(/*seed=*/0); + } + + BenchmarkGraph benchmark_graph = setup_compute_graph( + test_case, test_case.operator_name(), /*op_invocations_per_execute=*/1); + ComputeGraph& graph = *benchmark_graph.graph; + graph.prepare(); + graph.prepack(); + + const std::vector upper_input_sizes = test_case.inputs().at(0).sizes; + const std::vector shrunk_input_sizes = {1, 64, 20, 51}; + const std::vector shrunk_output_sizes = {1, 4, 20, 51}; + TestCase shrunk_case = test_case; + shrunk_case.inputs().at(0).sizes = shrunk_input_sizes; + shrunk_case.inputs().at(0).resize_data(1 * 64 * 20 * 51); + shrunk_case.outputs().at(0).sizes = shrunk_output_sizes; + shrunk_case.outputs().at(0).resize_data(1 * 4 * 20 * 51); + reference_impl(shrunk_case); + + constexpr int kRepetitions = 4; + for (int repetition = 0; repetition < kRepetitions; ++repetition) { + graph.resize_input(0, upper_input_sizes); + graph.propagate_resize(); + graph.maybe_cast_and_copy_into_staging( + graph.inputs().at(0).staging, + test_case.inputs().at(0).get_data_ptr(), + test_case.inputs().at(0).numel(), + vkapi::kFloat); + graph.execute(); + + graph.resize_input(0, shrunk_input_sizes); + graph.propagate_resize(); + if (graph.sizes_of(graph.outputs().at(0).value) != shrunk_output_sizes) { + throw std::runtime_error("streaming im2col output did not shrink"); + } + graph.maybe_cast_and_copy_into_staging( + graph.inputs().at(0).staging, + shrunk_case.inputs().at(0).get_data_ptr(), + shrunk_case.inputs().at(0).numel(), + vkapi::kFloat); + graph.execute(); + graph.maybe_cast_and_copy_from_staging( + graph.outputs().at(0).staging, + shrunk_case.outputs().at(0).get_mutable_data_ptr(), + shrunk_case.outputs().at(0).numel(), + vkapi::kFloat); + if (!shrunk_case.outputs().at(0).validate_against_reference( + shrunk_case.get_abs_tolerance(), shrunk_case.get_rel_tolerance())) { + throw std::runtime_error("streaming im2col shrink output was stale"); + } + } + + const Q8taConv2dStreamPlan upper_plan = make_q8ta_conv2d_stream_plan( + /*batch=*/10, + /*flattened_kernel_size=*/576, + /*out_height=*/30, + /*out_width=*/99, + kQ8taConv2dIm2ColScratchBudgetBytes); + if (!upper_plan.feasible || upper_plan.num_tiles != 2 || + upper_plan.rows_per_tile <= 20) { + throw std::runtime_error("dynamic shrink test did not create dead tiles"); + } + const int64_t resized_scratch_bytes = + 576 * upper_plan.rows_per_tile * utils::align_up_4(51); + if (resized_scratch_bytes > kQ8taConv2dIm2ColScratchBudgetBytes) { + throw std::runtime_error("streaming im2col scratch exceeded its cap"); + } + std::cout << "Streaming im2col dynamic shrink PASSED" << std::endl; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "streaming_dynamic_shrink") { + TestCaseSet set; + set.custom_test = execute_streaming_dynamic_shrink_test; + return set; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_im2col_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_im2col_set.cpp new file mode 100644 index 00000000000..b94fbc25140 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_im2col_set.cpp @@ -0,0 +1,138 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +struct StreamingIm2colConfig { + Conv2dConfig config; + std::vector int8_memory_layouts; + std::string impl_selector; +}; + +std::vector generate_streaming_im2col_test_cases( + const std::vector& configs) { + const bool has_int8_dot_product = + vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product(); + std::vector test_cases; + for (const StreamingIm2colConfig& streaming_config : configs) { + // Forced-fallback cases need no int8 dot-product support, so they run on + // all devices; the other routes stay gated. + if (!has_int8_dot_product && + streaming_config.impl_selector != "im2col_fallback") { + continue; + } + Conv2dConfig config = streaming_config.config; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = + make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); + for (const utils::GPUMemoryLayout int8_memory_layout : + streaming_config.int8_memory_layouts) { + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + int8_memory_layout, + streaming_config.impl_selector, + /*im2col_options=*/nullptr, + /*input_scale_val=*/1.0f, + /*input_data_gen=*/DataGenType::RANDINT)); + } + } + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "streaming_im2col") { + const std::vector configs = { + // Full fit + {{OutInChannels(4, 32), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 10}, + {utils::kPackedInt8_4W4C}, + "im2col_fallback"}, + // Streaming + {{OutInChannels(4, 64), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 10}, + {utils::kPackedInt8_4W4C}, + "im2col_fallback"}, + // Full fit + {{OutInChannels(4, 32), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 10}, + {utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C}, + "im2col_auto"}, + // Streaming + {{OutInChannels(4, 64), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 10}, + {utils::kPackedInt8_4C1W}, + "im2col_auto"}, + // Grouped + {{OutInChannels(16, 32), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2, + 20}, + {utils::kPackedInt8_4W4C}, + "im2col_auto"}, + // Output channel tail + {{OutInChannels(10, 32), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 20}, + {utils::kPackedInt8_4W4C}, + "im2col_auto"}, + }; + return { + generate_streaming_im2col_test_cases(configs), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/unsigned_dot_selection_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/unsigned_dot_selection_set.cpp new file mode 100644 index 00000000000..0f568ad30c9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/unsigned_dot_selection_set.cpp @@ -0,0 +1,43 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +void check_unsigned_dot_selection() { + const vkapi::Adapter& adapter = *vkcompute::api::context()->adapter_ptr(); + const bool prefers_unsigned_dot = + adapter.accelerates_unsigned_packed4x8_dot() && + !adapter.accelerates_signed_packed4x8_dot(); + VK_CHECK_COND( + can_use_unsigned_pw_dot(adapter, kMaxUnsignedDotAccumulatorBytes) == + prefers_unsigned_dot); + VK_CHECK_COND( + !can_use_unsigned_pw_dot(adapter, kMaxUnsignedDotAccumulatorBytes + 1)); +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "unsigned_dot_selection") { + TestCaseSet set; + set.custom_test = check_unsigned_dot_selection; + return set; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/test_q8ta_conv2d_transposed.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/conv2d_transposed.cpp similarity index 71% rename from backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/test_q8ta_conv2d_transposed.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/conv2d_transposed.cpp index 4ead8123f45..18c2cd16835 100644 --- a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/test_q8ta_conv2d_transposed.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/conv2d_transposed.cpp @@ -4,45 +4,43 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include - -#include -#include - -#include - -#include -#include +#include -// #define DEBUG_MODE - -using namespace executorch::vulkan::prototyping; +#include +#include +#include +#include +#include -using namespace vkcompute; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d_transposed { -static constexpr int64_t kRefDimSizeLimit = 100; +namespace { // Transposed convolution output size formula: // H_out = (H_in - 1) * stride_h - 2 * pad_h + dilation_h * (K_h - 1) // + output_pad_h + 1 -static int64_t get_transpose_output_height( +int64_t get_transpose_output_height( const Conv2dConfig& config, int32_t output_pad_h) { return (config.input_size.h - 1) * config.stride.h - 2 * config.padding.h + config.dilation.h * (config.kernel.h - 1) + output_pad_h + 1; } -static int64_t get_transpose_output_width( +int64_t get_transpose_output_width( const Conv2dConfig& config, int32_t output_pad_w) { return (config.input_size.w - 1) * config.stride.w - 2 * config.padding.w + config.dilation.w * (config.kernel.w - 1) + output_pad_w + 1; } +} // namespace + // Utility function to create a test case from a Conv2dConfig for transposed // convolution -static TestCase create_test_case_from_config( +TestCase create_test_case_from_config( const Conv2dConfig& config, int32_t output_pad_h, int32_t output_pad_w, @@ -209,163 +207,31 @@ static TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for debugging -std::vector generate_quantized_conv2d_transposed_easy_cases() { - std::vector test_cases; - - Conv2dConfig config = { - OutInChannels(16, 32), - InputSize2D(8, 8), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1, - }; - - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - for (const utils::GPUMemoryLayout int8_memory_layout : int8_memory_layouts) { - config.test_case_name = - make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - /*output_pad_h=*/1, - /*output_pad_w=*/1, - vkapi::kFloat, - utils::kTexture3D, - int8_memory_layout)); - } +int64_t quantized_conv2d_transposed_flop_calculator(const TestCase& test_case) { + int kernel_idx = 9; - return test_cases; -} + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); + const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); -// Generate test cases for quantized transposed conv2d -static std::vector generate_quantized_conv2d_transposed_test_cases() { - std::vector test_cases; - if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { - return test_cases; - } + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t C_out = output_sizes[1]; + int64_t K_h = kernel_sizes[0]; + int64_t K_w = kernel_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; - // Each entry: {config, output_pad_h, output_pad_w} - struct TransposedConvTestConfig { - Conv2dConfig config; - int32_t output_pad_h; - int32_t output_pad_w; - }; - - std::vector configs = { - // Basic transposed conv (stride=2, common in decoder networks) - {{OutInChannels(16, 32), - InputSize2D(8, 8), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - 1, - 1}, - {{OutInChannels(32, 64), - InputSize2D(4, 4), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - 1, - 1}, - // No output padding - {{OutInChannels(16, 32), - InputSize2D(8, 8), - KernelSize(4, 4), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - 0, - 0}, - // Stride=1 (degenerate case) - {{OutInChannels(16, 16), - InputSize2D(8, 8), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - 0, - 0}, - // Grouped transposed conv - {{OutInChannels(32, 64), - InputSize2D(8, 8), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 2}, - 1, - 1}, - // Larger spatial - {{OutInChannels(64, 128), - InputSize2D(16, 16), - KernelSize(4, 4), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - 0, - 0}, - // Performance cases - {{OutInChannels(64, 128), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - 1, - 1}, - {{OutInChannels(128, 256), - InputSize2D(16, 16), - KernelSize(4, 4), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - 0, - 0}, - }; - - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - for (auto& tc : configs) { - auto& config = tc.config; - bool is_performance = config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit || - config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - - for (const utils::GPUMemoryLayout int8_memory_layout : - int8_memory_layouts) { - config.test_case_name = make_test_case_name( - config, is_performance, utils::kTexture3D, utils::kBuffer); - - test_cases.push_back(create_test_case_from_config( - config, - tc.output_pad_h, - tc.output_pad_w, - vkapi::kFloat, - utils::kTexture3D, - int8_memory_layout)); - } - } + int64_t output_elements = N * C_out * H_out * W_out; + int64_t ops_per_output = C_in * K_h * K_w; - return test_cases; + return output_elements * ops_per_output; } +namespace { + // Reference implementation for quantized transposed conv2d -static void conv2d_transposed_q8ta_reference_impl(TestCase& test_case) { +void conv2d_transposed_q8ta_reference_impl(TestCase& test_case) { int32_t idx = 0; const ValueSpec& input_spec = test_case.inputs()[idx++]; const ValueSpec& input_scale_spec = test_case.inputs()[idx++]; @@ -544,61 +410,13 @@ static void conv2d_transposed_q8ta_reference_impl(TestCase& test_case) { } } -static void reference_impl(TestCase& test_case) { - conv2d_transposed_q8ta_reference_impl(test_case); -} - -static int64_t quantized_conv2d_transposed_flop_calculator( - const TestCase& test_case) { - int kernel_idx = 9; - - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t C_out = output_sizes[1]; - int64_t K_h = kernel_sizes[0]; - int64_t K_w = kernel_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - int64_t output_elements = N * C_out * H_out * W_out; - int64_t ops_per_output = C_in * K_h * K_w; +} // namespace - return output_elements * ops_per_output; +void reference_impl(TestCase& test_case) { + conv2d_transposed_q8ta_reference_impl(test_case); } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(true); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "Quantized Transposed Conv2d Operation with Output Quantization Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( -#ifdef DEBUG_MODE - generate_quantized_conv2d_transposed_easy_cases, -#else - generate_quantized_conv2d_transposed_test_cases, -#endif - quantized_conv2d_transposed_flop_calculator, - "QuantizedTransposedConv2d", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_conv2d_transposed +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/conv2d_transposed.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/conv2d_transposed.h new file mode 100644 index 00000000000..34873570947 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/conv2d_transposed.h @@ -0,0 +1,35 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d_transposed { + +constexpr int64_t kRefDimSizeLimit = 100; + +TestCase create_test_case_from_config( + const Conv2dConfig& config, + int32_t output_pad_h, + int32_t output_pad_w, + vkapi::ScalarType input_dtype, + utils::StorageType fp_storage_type, + utils::GPUMemoryLayout int8_memory_layout); + +void reference_impl(TestCase& test_case); + +int64_t quantized_conv2d_transposed_flop_calculator(const TestCase& test_case); + +} // namespace q8ta_conv2d_transposed +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/test_sets/conv2d_transposed_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/test_sets/conv2d_transposed_sets.cpp new file mode 100644 index 00000000000..46bdef86bcb --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/test_sets/conv2d_transposed_sets.cpp @@ -0,0 +1,196 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d_transposed { + +namespace { + +// Each entry: {config, output_pad_h, output_pad_w} +struct TransposedConvTestConfig { + Conv2dConfig config; + int32_t output_pad_h; + int32_t output_pad_w; +}; + +std::vector generate_test_cases( + const std::vector& configs, + const std::vector& int8_memory_layouts, + bool require_int8_dot_product = true) { + std::vector test_cases; + if (require_int8_dot_product && + !vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { + return test_cases; + } + + for (auto tc : configs) { + auto& config = tc.config; + bool is_performance = config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit || + config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + + for (const utils::GPUMemoryLayout int8_memory_layout : + int8_memory_layouts) { + config.test_case_name = make_test_case_name( + config, is_performance, utils::kTexture3D, utils::kBuffer); + + test_cases.push_back(create_test_case_from_config( + config, + tc.output_pad_h, + tc.output_pad_w, + vkapi::kFloat, + utils::kTexture3D, + int8_memory_layout)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d_transposed", "correctness") { + const std::vector configs = { + // Basic transposed conv (stride=2, common in decoder networks) + {{OutInChannels(16, 32), + InputSize2D(8, 8), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + 1, + 1}, + {{OutInChannels(32, 64), + InputSize2D(4, 4), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + 1, + 1}, + // No output padding + {{OutInChannels(16, 32), + InputSize2D(8, 8), + KernelSize(4, 4), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + 0, + 0}, + // Stride=1 (degenerate case) + {{OutInChannels(16, 16), + InputSize2D(8, 8), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + 0, + 0}, + // Grouped transposed conv + {{OutInChannels(32, 64), + InputSize2D(8, 8), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 2}, + 1, + 1}, + }; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4C, + }; + return { + generate_test_cases(configs, int8_memory_layouts), + reference_impl, + quantized_conv2d_transposed_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8ta/conv2d_transposed", "debug") { + const std::vector configs = { + // Easy test case for debugging + {{OutInChannels(16, 32), + InputSize2D(8, 8), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + /*output_pad_h=*/1, + /*output_pad_w=*/1}, + }; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4C, + }; + return { + generate_test_cases( + configs, int8_memory_layouts, /*require_int8_dot_product=*/false), + reference_impl, + quantized_conv2d_transposed_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8ta/conv2d_transposed", "performance") { + const std::vector configs = { + // Larger spatial + {{OutInChannels(64, 128), + InputSize2D(16, 16), + KernelSize(4, 4), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + 0, + 0}, + // Performance cases + {{OutInChannels(64, 128), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + 1, + 1}, + {{OutInChannels(128, 256), + InputSize2D(16, 16), + KernelSize(4, 4), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + 0, + 0}, + }; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4C, + }; + return { + generate_test_cases(configs, int8_memory_layouts), + reference_impl, + quantized_conv2d_transposed_flop_calculator}; +} + +} // namespace q8ta_conv2d_transposed +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/test_q8ta_linear.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/linear.cpp similarity index 74% rename from backends/vulkan/test/ops/benchmark/cases/q8ta/linear/test_q8ta_linear.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/linear/linear.cpp index a935508bc19..3629d293eb0 100644 --- a/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/test_q8ta_linear.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/linear.cpp @@ -4,34 +4,23 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include - -#include -#include - -#include - -#include - -using namespace executorch::vulkan::prototyping; - -using namespace vkcompute; +#include -static constexpr int64_t kRefDimSizeLimit = 300; +#include +#include +#include +#include +#include -struct LinearConfig { - int64_t M; - int64_t K; - int64_t N; - bool has_bias = true; - std::string test_case_name = "placeholder"; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_linear { -static TestCase create_test_case_from_config( +TestCase create_test_case_from_config( const LinearConfig& config, vkapi::ScalarType input_dtype, - const std::string& impl_selector = "") { + const std::string& impl_selector) { TestCase test_case; bool is_perf = config.M >= kRefDimSizeLimit || config.K >= kRefDimSizeLimit || @@ -172,63 +161,26 @@ static TestCase create_test_case_from_config( return test_case; } -// Generate test cases for q8ta_linear operation -static std::vector generate_q8ta_linear_test_cases() { - std::vector test_cases; - - std::vector configs = { - // Batch size 1 cases (test both tiled and gemv) - {1, 64, 32}, - {1, 128, 64}, - {1, 256, 128}, - {1, 128, 64, false}, - // Multi-batch cases - {4, 64, 32}, - {4, 128, 64}, - {4, 256, 128}, - {32, 64, 32}, - {32, 128, 64}, - {32, 256, 128}, - // No bias tests - {32, 128, 64, false}, - {32, 256, 128, false}, - // Performance cases - {1, 512, 512}, - {1, 2048, 2048}, - {1, 512, 9059}, - {256, 2048, 2048}, - {512, 2048, 2048}, - {1024, 2048, 2048}, - }; - - for (auto config : configs) { - bool is_performance = config.M >= kRefDimSizeLimit || - config.K >= kRefDimSizeLimit || config.N >= kRefDimSizeLimit; - - std::string prefix = is_performance ? "performance_" : "correctness_"; - std::string generated_test_case_name = prefix + std::to_string(config.M) + - "_" + std::to_string(config.K) + "_" + std::to_string(config.N); - if (!config.has_bias) { - generated_test_case_name += "_no_bias"; - } +int64_t q8ta_linear_flop_calculator(const TestCase& test_case) { + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& weight_sizes = test_case.inputs()[3].get_tensor_sizes(); - config.test_case_name = generated_test_case_name; + int64_t batch_size = input_sizes[0]; + int64_t in_features = input_sizes[1]; + int64_t out_features = weight_sizes[0]; - // Default (tiled) variant - test_cases.push_back(create_test_case_from_config(config, vkapi::kFloat)); + int64_t output_elements = batch_size * out_features; + int64_t ops_per_output = in_features; - // For batch size 1, also test the gemv variant - if (config.M == 1) { - test_cases.push_back( - create_test_case_from_config(config, vkapi::kFloat, "gemv")); - } - } + int64_t flop = output_elements * ops_per_output; - return test_cases; + return flop; } +namespace { + // Reference implementation for q8ta_linear (activation+weight+output quantized) -static void q8ta_linear_reference_impl(TestCase& test_case) { +void q8ta_linear_reference_impl(TestCase& test_case) { int32_t idx = 0; const ValueSpec& input_spec = test_case.inputs()[idx++]; const ValueSpec& input_scale_spec = test_case.inputs()[idx++]; @@ -329,45 +281,13 @@ static void q8ta_linear_reference_impl(TestCase& test_case) { } } -static void reference_impl(TestCase& test_case) { - q8ta_linear_reference_impl(test_case); -} - -static int64_t q8ta_linear_flop_calculator(const TestCase& test_case) { - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& weight_sizes = test_case.inputs()[3].get_tensor_sizes(); - - int64_t batch_size = input_sizes[0]; - int64_t in_features = input_sizes[1]; - int64_t out_features = weight_sizes[0]; - - int64_t output_elements = batch_size * out_features; - int64_t ops_per_output = in_features; - - int64_t flop = output_elements * ops_per_output; +} // namespace - return flop; +void reference_impl(TestCase& test_case) { + q8ta_linear_reference_impl(test_case); } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Q8ta Linear Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_q8ta_linear_test_cases, - q8ta_linear_flop_calculator, - "Q8taLinear", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/linear.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/linear.h new file mode 100644 index 00000000000..0f27844ff43 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/linear.h @@ -0,0 +1,41 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_linear { + +constexpr int64_t kRefDimSizeLimit = 300; + +struct LinearConfig { + int64_t M; + int64_t K; + int64_t N; + bool has_bias = true; + std::string test_case_name = "placeholder"; +}; + +TestCase create_test_case_from_config( + const LinearConfig& config, + vkapi::ScalarType input_dtype, + const std::string& impl_selector = ""); + +void reference_impl(TestCase& test_case); + +int64_t q8ta_linear_flop_calculator(const TestCase& test_case); + +} // namespace q8ta_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/test_sets/linear_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/test_sets/linear_sets.cpp new file mode 100644 index 00000000000..1483728fc47 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/test_sets/linear_sets.cpp @@ -0,0 +1,96 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_linear { + +namespace { + +std::vector generate_test_cases( + const std::vector& configs) { + std::vector test_cases; + + for (auto config : configs) { + bool is_performance = config.M >= kRefDimSizeLimit || + config.K >= kRefDimSizeLimit || config.N >= kRefDimSizeLimit; + + std::string prefix = is_performance ? "performance_" : "correctness_"; + std::string generated_test_case_name = prefix + std::to_string(config.M) + + "_" + std::to_string(config.K) + "_" + std::to_string(config.N); + if (!config.has_bias) { + generated_test_case_name += "_no_bias"; + } + + config.test_case_name = generated_test_case_name; + + // Default (tiled) variant + test_cases.push_back(create_test_case_from_config(config, vkapi::kFloat)); + + // For batch size 1, also test the gemv variant + if (config.M == 1) { + test_cases.push_back( + create_test_case_from_config(config, vkapi::kFloat, "gemv")); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/linear", "correctness") { + const std::vector configs = { + // Batch size 1 cases (test both tiled and gemv) + {1, 64, 32}, + {1, 128, 64}, + {1, 256, 128}, + {1, 128, 64, false}, + // Multi-batch cases + {4, 64, 32}, + {4, 128, 64}, + {4, 256, 128}, + {32, 64, 32}, + {32, 128, 64}, + {32, 256, 128}, + // No bias tests + {32, 128, 64, false}, + {32, 256, 128, false}, + }; + + return { + generate_test_cases(configs), + reference_impl, + q8ta_linear_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8ta/linear", "performance") { + const std::vector configs = { + // Performance cases + {1, 512, 512}, + {1, 2048, 2048}, + {1, 512, 9059}, + {256, 2048, 2048}, + {512, 2048, 2048}, + {1024, 2048, 2048}, + }; + + return { + generate_test_cases(configs), + reference_impl, + q8ta_linear_flop_calculator}; +} + +} // namespace q8ta_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/test_q8ta_pixel_shuffle.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/pixel_shuffle.cpp similarity index 56% rename from backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/test_q8ta_pixel_shuffle.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/pixel_shuffle.cpp index 948ee347520..5f805171fac 100644 --- a/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/test_q8ta_pixel_shuffle.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/pixel_shuffle.cpp @@ -4,41 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include -#include +#include + #include #include -#include -#include +#include +#include #include -#include - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 512; - -// Test op signature in TestQ8taPixelShuffle.cpp: -// test_q8ta_pixel_shuffle(fp_in, in_scale, in_zp, out_scale, out_zp, -// upscale_factor, in_layout, out_layout) -> fp_out -// Implementation: fused fast-path kernel. The in_layout / out_layout strings -// select the channels-packed int8x4 layout used for the temporary quantized -// tensors. Supported values: "4W4C", "4C1W". -// (PACKED_INT8_CONV2D is a Python/serialization-level alias that the runtime -// resolves to kPackedInt8_4C1W, so it is not exercised separately here -- it -// would only re-test the same C++ kernel path as "4C1W".) - -struct PixelShuffleConfig { - std::vector in_shape; // [N, C*r*r, H, W] - int upscale_factor; - bool same_qparams; // if true, in_scale == out_scale and in_zp == out_zp - std::string in_layout = "4W4C"; - std::string out_layout = "4W4C"; - std::string test_case_name = "ACCU"; - std::string op_name = "test_q8ta_pixel_shuffle"; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_pixel_shuffle { TestCase create_test_case_from_config(const PixelShuffleConfig& config) { TestCase test_case; @@ -124,92 +101,6 @@ TestCase create_test_case_from_config(const PixelShuffleConfig& config) { return test_case; } -// All (in_layout, out_layout) pairs across the channels-packed int8 family. -// CONV2D is a Python-level alias that resolves to 4C1W at the C++ runtime, so -// it is not listed separately -- it would just re-run the 4C1W kernel path. -static const std::vector>& -get_layout_pairs() { - static const std::vector> layout_pairs = { - {"4W4C", "4W4C"}, - {"4W4C", "4C1W"}, - {"4C1W", "4W4C"}, - {"4C1W", "4C1W"}, - }; - return layout_pairs; -} - -std::vector generate_correctness_cases() { - std::vector test_cases; - - // Small shapes, all use r=2 (the only factor needed by the model). - // Shape format is the *input* shape [N, C_in, H, W] where C_in = C_out * r*r. - std::vector> shapes = { - // Small even W to be a multiple of 4 after upscaling. - {1, 16, 4, 4}, // out: [1, 4, 8, 8] - {1, 24, 8, 4}, // out: [1, 6, 16, 8] - {1, 32, 12, 8}, // out: [1, 8, 24, 16] - {1, 96, 16, 9}, // out: [1, 24, 32, 18] - first model shape - }; - - for (const auto& shape : shapes) { - for (bool same_qp : {true, false}) { - for (const auto& layouts : get_layout_pairs()) { - PixelShuffleConfig cfg; - cfg.in_shape = shape; - cfg.upscale_factor = 2; - cfg.same_qparams = same_qp; - cfg.in_layout = layouts.first; - cfg.out_layout = layouts.second; - cfg.test_case_name = "ACCU"; - - test_cases.push_back(create_test_case_from_config(cfg)); - } - } - } - - return test_cases; -} - -std::vector generate_perf_cases() { - std::vector test_cases; - - // Model perf shapes (output shapes from the RefineNet decoder; we compute - // the input shape as [N, C_out * r*r, H_out / r, W_out / r]). - // Output shapes: [1, 24, 32, 18], [1, 24, 64, 36], [1, 24, 128, 72], - // [1, 24, 256, 144]. For r=2, in shapes = [1, 96, 16, 9], etc. - std::vector> in_shapes = { - {1, 96, 16, 9}, - {1, 96, 32, 18}, - {1, 96, 64, 36}, - {1, 96, 128, 72}, - }; - - for (const auto& shape : in_shapes) { - for (const auto& layouts : get_layout_pairs()) { - PixelShuffleConfig cfg; - cfg.in_shape = shape; - cfg.upscale_factor = 2; - cfg.same_qparams = true; // residual-style: scales match - cfg.in_layout = layouts.first; - cfg.out_layout = layouts.second; - cfg.test_case_name = "PERF"; - - test_cases.push_back(create_test_case_from_config(cfg)); - } - } - - return test_cases; -} - -std::vector generate_all_cases() { - std::vector all = generate_correctness_cases(); - std::vector perf = generate_perf_cases(); - for (auto& tc : perf) { - all.push_back(tc); - } - return all; -} - // Reference: quantize input, do PyTorch-equivalent pixel shuffle, requantize, // then dequantize for comparison. void q8ta_pixel_shuffle_reference_impl(TestCase& test_case) { @@ -288,24 +179,7 @@ void q8ta_pixel_shuffle_reference_impl(TestCase& test_case) { } } -int main(int /*argc*/, char* /*argv*/[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Q8TA PixelShuffle Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = q8ta_pixel_shuffle_reference_impl; - - auto results = execute_test_cases( - generate_all_cases, - "Q8taPixelShuffle", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_pixel_shuffle +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/pixel_shuffle.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/pixel_shuffle.h new file mode 100644 index 00000000000..42dcc329359 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/pixel_shuffle.h @@ -0,0 +1,49 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_pixel_shuffle { + +constexpr int64_t kRefDimSizeLimit = 512; + +// Test op signature in TestQ8taPixelShuffle.cpp: +// test_q8ta_pixel_shuffle(fp_in, in_scale, in_zp, out_scale, out_zp, +// upscale_factor, in_layout, out_layout) -> fp_out +// Implementation: fused fast-path kernel. The in_layout / out_layout strings +// select the channels-packed int8x4 layout used for the temporary quantized +// tensors. Supported values: "4W4C", "4C1W". +// (PACKED_INT8_CONV2D is a Python/serialization-level alias that the runtime +// resolves to kPackedInt8_4C1W, so it is not exercised separately here -- it +// would only re-test the same C++ kernel path as "4C1W".) + +struct PixelShuffleConfig { + std::vector in_shape; // [N, C*r*r, H, W] + int upscale_factor; + bool same_qparams; // if true, in_scale == out_scale and in_zp == out_zp + std::string in_layout = "4W4C"; + std::string out_layout = "4W4C"; + std::string test_case_name = "ACCU"; + std::string op_name = "test_q8ta_pixel_shuffle"; +}; + +TestCase create_test_case_from_config(const PixelShuffleConfig& config); + +void q8ta_pixel_shuffle_reference_impl(TestCase& test_case); + +} // namespace q8ta_pixel_shuffle +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/test_sets/pixel_shuffle_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/test_sets/pixel_shuffle_sets.cpp new file mode 100644 index 00000000000..13605714009 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/test_sets/pixel_shuffle_sets.cpp @@ -0,0 +1,96 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_pixel_shuffle { + +namespace { + +// All (in_layout, out_layout) pairs across the channels-packed int8 family. +// CONV2D is a Python-level alias that resolves to 4C1W at the C++ runtime, so +// it is not listed separately -- it would just re-run the 4C1W kernel path. +const std::vector> kLayoutPairs = { + {"4W4C", "4W4C"}, + {"4W4C", "4C1W"}, + {"4C1W", "4W4C"}, + {"4C1W", "4C1W"}, +}; + +std::vector generate_test_cases( + const std::vector>& in_shapes, + int upscale_factor, + const std::vector& same_qparams, + const std::string& test_case_name) { + std::vector test_cases; + + for (const auto& shape : in_shapes) { + for (bool same_qp : same_qparams) { + for (const auto& layouts : kLayoutPairs) { + PixelShuffleConfig cfg; + cfg.in_shape = shape; + cfg.upscale_factor = upscale_factor; + cfg.same_qparams = same_qp; + cfg.in_layout = layouts.first; + cfg.out_layout = layouts.second; + cfg.test_case_name = test_case_name; + + test_cases.push_back(create_test_case_from_config(cfg)); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/pixel_shuffle", "correctness") { + // Small shapes, all use r=2 (the only factor needed by the model). + // Shape format is the *input* shape [N, C_in, H, W] where C_in = C_out * r*r. + const std::vector> in_shapes = { + // Small even W to be a multiple of 4 after upscaling. + {1, 16, 4, 4}, // out: [1, 4, 8, 8] + {1, 24, 8, 4}, // out: [1, 6, 16, 8] + {1, 32, 12, 8}, // out: [1, 8, 24, 16] + {1, 96, 16, 9}, // out: [1, 24, 32, 18] - first model shape + }; + const std::vector same_qparams = {true, false}; + return { + generate_test_cases( + in_shapes, /*upscale_factor=*/2, same_qparams, "ACCU"), + q8ta_pixel_shuffle_reference_impl}; +} + +REGISTER_TEST_CASE_SET("q8ta/pixel_shuffle", "performance") { + // Model perf shapes (output shapes from the RefineNet decoder; we compute + // the input shape as [N, C_out * r*r, H_out / r, W_out / r]). + // Output shapes: [1, 24, 32, 18], [1, 24, 64, 36], [1, 24, 128, 72], + // [1, 24, 256, 144]. For r=2, in shapes = [1, 96, 16, 9], etc. + const std::vector> in_shapes = { + {1, 96, 16, 9}, + {1, 96, 32, 18}, + {1, 96, 64, 36}, + {1, 96, 128, 72}, + }; + const std::vector same_qparams = {true}; // residual-style: scales match + return { + generate_test_cases( + in_shapes, /*upscale_factor=*/2, same_qparams, "PERF"), + q8ta_pixel_shuffle_reference_impl}; +} + +} // namespace q8ta_pixel_shuffle +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.cpp new file mode 100644 index 00000000000..b6ad788b9db --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.cpp @@ -0,0 +1,164 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_qdq { + +// Utility function to create a test case from a QDQ8BitConfig +TestCase create_test_case_from_config( + const QDQ8BitConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype, + utils::GPUMemoryLayout fp_memory_layout, + utils::GPUMemoryLayout quantized_memory_layout, + const std::string& impl_selector) { + TestCase test_case; + + // Create a descriptive name for the test case + // QDQ: fp input -> int8 staging -> fp output (label as f32->f32) + std::string prefix = config.test_case_name; // "ACCU" or "PERF" + std::string storage_str = repr_str(storage_type, fp_memory_layout) + "->" + + repr_str(utils::kBuffer, quantized_memory_layout); + std::string suffix = impl_selector.empty() ? "" : "[" + impl_selector + "]"; + std::string test_name = make_test_label( + prefix, + dtype_short(input_dtype), + dtype_short(input_dtype), + shape_bracket(config.shape), + storage_str, + suffix); + test_case.set_name(test_name); + + // Set the operator name for the test case + std::string operator_name = "test_etvk." + config.op_name + ".default"; + test_case.set_operator_name(operator_name); + + // Input tensor (float) - any dimensionality + ValueSpec input_tensor( + config.shape, + input_dtype, + storage_type, + fp_memory_layout, + DataGenType::RANDOM); + + float scale_val = 0.007112; + ValueSpec scale(scale_val); + + // Generate random zero point within quantization range + int32_t zero_point_val = 0; + ValueSpec zero_point(zero_point_val); + + // GPUMemoryLayout as integer (will be cast in the operator) + int32_t layout_int = static_cast(quantized_memory_layout); + ValueSpec layout_spec(layout_int); + + // impl_selector string + ValueSpec impl_selector_spec = ValueSpec::make_string(impl_selector); + + // Output tensor (float) - same shape as input + ValueSpec output_tensor( + config.shape, + input_dtype, + storage_type, + fp_memory_layout, + DataGenType::ZEROS); + + // Add all specs to test case + test_case.add_input_spec(input_tensor); + test_case.add_input_spec(scale); + test_case.add_input_spec(zero_point); + test_case.add_input_spec(layout_spec); + test_case.add_input_spec(impl_selector_spec); + test_case.add_output_spec(output_tensor); + + test_case.set_abs_tolerance(scale_val + 1e-4); + + // Use layout-only filter for this test since quantize/dequantize ARE the + // operations being tested, not overhead + test_case.set_shader_filter(kLayoutOnlyShaderFilter); + + return test_case; +} + +// Reference implementation for q_dq_8bit operation +void q_dq_8bit_reference_impl(TestCase& test_case) { + int32_t idx = 0; + const ValueSpec& input_spec = test_case.inputs()[idx++]; + const ValueSpec& scale_spec = test_case.inputs()[idx++]; + const ValueSpec& zero_point_spec = test_case.inputs()[idx++]; + const ValueSpec& layout_spec = test_case.inputs()[idx++]; + (void)layout_spec; // Not used in reference implementation + + // Extract output specification + ValueSpec& output_spec = test_case.outputs()[0]; + + // Get tensor dimensions (arbitrary dimensionality) + auto input_sizes = input_spec.get_tensor_sizes(); + + // Calculate total number of elements + int64_t num_elements = 1; + for (const auto& dim : input_sizes) { + num_elements *= dim; + } + + // Skip for large tensors since computation time will be extremely slow + for (const auto& dim : input_sizes) { + if (dim > kRefDimSizeLimit) { + throw std::invalid_argument( + "One or more dimensions exceed the allowed limit for reference implementation."); + } + } + + if (input_spec.dtype != vkapi::kFloat) { + throw std::invalid_argument("Unsupported dtype"); + } + + // Get raw data pointers + auto& input_data = input_spec.get_float_data(); + + // Extract the randomized scale and zero point values + float scale = scale_spec.get_float_value(); + int32_t zero_point = zero_point_spec.get_int_value(); + int32_t quant_min = -128; + int32_t quant_max = 127; + + // Prepare output data + auto& ref_data = output_spec.get_ref_float_data(); + ref_data.resize(num_elements); + + // Perform quantize-dequantize operation on each element + for (int64_t i = 0; i < num_elements; ++i) { + float input_val = input_data[i]; + + // Quantize: quantized = round(input / scale + zero_point) + float quantized_float = std::round(input_val / scale) + zero_point; + + // Clamp to quantization range + quantized_float = std::max(quantized_float, static_cast(quant_min)); + quantized_float = std::min(quantized_float, static_cast(quant_max)); + + int32_t quantized_int = static_cast(quantized_float); + + // Dequantize: output = (quantized - zero_point) * scale + float dequantized = (quantized_int - zero_point) * scale; + + ref_data[i] = dequantized; + } +} + +} // namespace q8ta_qdq +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.h new file mode 100644 index 00000000000..7b63610edff --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.h @@ -0,0 +1,42 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_qdq { + +constexpr int64_t kRefDimSizeLimit = 512; + +// Configuration struct for tensor quantize-dequantize testing +struct QDQ8BitConfig { + std::vector shape; // Tensor shape (can be any dimensionality) + std::string test_case_name = "placeholder"; + std::string op_name = "q_dq_8bit_per_tensor"; +}; + +TestCase create_test_case_from_config( + const QDQ8BitConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype, + utils::GPUMemoryLayout fp_memory_layout, + utils::GPUMemoryLayout quantized_memory_layout, + const std::string& impl_selector = ""); + +void q_dq_8bit_reference_impl(TestCase& test_case); + +} // namespace q8ta_qdq +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_q8ta_qdq.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_q8ta_qdq.cpp deleted file mode 100644 index d11988dd192..00000000000 --- a/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_q8ta_qdq.cpp +++ /dev/null @@ -1,381 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include -#include -#include -#include -#include -#include - -#include - -// #define DEBUG_MODE - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 512; - -// Configuration struct for tensor quantize-dequantize testing -struct QDQ8BitConfig { - std::vector shape; // Tensor shape (can be any dimensionality) - std::string test_case_name = "placeholder"; - std::string op_name = "q_dq_8bit_per_tensor"; -}; - -// Utility function to create a test case from a QDQ8BitConfig -TestCase create_test_case_from_config( - const QDQ8BitConfig& config, - utils::StorageType storage_type, - vkapi::ScalarType input_dtype, - utils::GPUMemoryLayout fp_memory_layout, - utils::GPUMemoryLayout quantized_memory_layout, - const std::string& impl_selector = "") { - TestCase test_case; - - // Create a descriptive name for the test case - // QDQ: fp input -> int8 staging -> fp output (label as f32->f32) - std::string prefix = config.test_case_name; // "ACCU" or "PERF" - std::string storage_str = repr_str(storage_type, fp_memory_layout) + "->" + - repr_str(utils::kBuffer, quantized_memory_layout); - std::string suffix = impl_selector.empty() ? "" : "[" + impl_selector + "]"; - std::string test_name = make_test_label( - prefix, - dtype_short(input_dtype), - dtype_short(input_dtype), - shape_bracket(config.shape), - storage_str, - suffix); - test_case.set_name(test_name); - - // Set the operator name for the test case - std::string operator_name = "test_etvk." + config.op_name + ".default"; - test_case.set_operator_name(operator_name); - - // Input tensor (float) - any dimensionality - ValueSpec input_tensor( - config.shape, - input_dtype, - storage_type, - fp_memory_layout, - DataGenType::RANDOM); - - float scale_val = 0.007112; - ValueSpec scale(scale_val); - - // Generate random zero point within quantization range - int32_t zero_point_val = 0; - ValueSpec zero_point(zero_point_val); - - // GPUMemoryLayout as integer (will be cast in the operator) - int32_t layout_int = static_cast(quantized_memory_layout); - ValueSpec layout_spec(layout_int); - - // impl_selector string - ValueSpec impl_selector_spec = ValueSpec::make_string(impl_selector); - - // Output tensor (float) - same shape as input - ValueSpec output_tensor( - config.shape, - input_dtype, - storage_type, - fp_memory_layout, - DataGenType::ZEROS); - - // Add all specs to test case - test_case.add_input_spec(input_tensor); - test_case.add_input_spec(scale); - test_case.add_input_spec(zero_point); - test_case.add_input_spec(layout_spec); - test_case.add_input_spec(impl_selector_spec); - test_case.add_output_spec(output_tensor); - - test_case.set_abs_tolerance(scale_val + 1e-4); - - // Use layout-only filter for this test since quantize/dequantize ARE the - // operations being tested, not overhead - test_case.set_shader_filter(kLayoutOnlyShaderFilter); - - return test_case; -} - -// Generate easy test cases for q_dq_8bit operation (for debugging) -std::vector generate_q_dq_8bit_easy_cases() { - std::vector test_cases; - - std::vector> shapes = { - {1, 16, 16, 16}, // 4D: [N, C, H, W] - {1, 144}, // 2D: exercises block config with ndim < 4 - {1, 90}, // 2D: matches skin_seg model's keypoint/bbox tensor sizes - {2, 1, 3, 16, 16}, // 5D: exercises high-dim batch support - {2, 1, 1, 4, 8, 16}, // 6D: exercises high-dim batch support - }; - - // FP memory layouts to test - std::vector fp_layouts = { - utils::kWidthPacked, - utils::kChannelsPacked, - }; - - // Quantized memory layouts to test - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - std::vector storage_types = {utils::kBuffer}; - std::vector float_types = {vkapi::kFloat}; - - // Generate test cases for each combination - for (const auto& shape : shapes) { - QDQ8BitConfig config = {shape, "ACCU"}; - for (const auto& fp_layout : fp_layouts) { - for (const auto& quant_layout : quant_layouts) { - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - test_cases.push_back(create_test_case_from_config( - config, storage_type, input_dtype, fp_layout, quant_layout)); - // For 4W4C layout, also test with legacy implementation - if (quant_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - storage_type, - input_dtype, - fp_layout, - quant_layout, - /*impl_selector=*/"legacy_4w4c")); - } - } - } - } - } - } - - return test_cases; -} - -// Generate test cases for q_dq_8bit operation -std::vector generate_q_dq_8bit_test_cases() { - std::vector test_cases; - - // Shapes to test (no layout specified - will be combined with all layouts) - std::vector> shapes = { - // 1D tensors - {144}, - {90}, - - // 2D tensors (exercises block config with ndim < 4) - {1, 144}, - {1, 90}, - {1, 4}, - {3, 32}, - - // 3D tensors - {1, 16, 32}, - {1, 3, 64}, - - // Small test cases for correctness - {1, 3, 16, 16}, - {1, 8, 32, 32}, - {1, 16, 24, 24}, - {1, 32, 12, 12}, - {1, 1, 64, 64}, - {1, 3, 64, 64}, - {1, 4, 16, 16}, - - // Different tensor sizes - {1, 8, 20, 20}, - {1, 16, 14, 14}, - {1, 8, 28, 28}, - - // Odd tensor sizes - {1, 3, 15, 15}, - {1, 13, 31, 31}, - {1, 17, 23, 23}, - - // Performance test cases (larger tensors) - {1, 64, 128, 128}, - {1, 32, 64, 64}, - {1, 128, 56, 56}, - - // 5D tensors (high-dim batch support) - {2, 1, 3, 16, 16}, - {1, 2, 8, 8, 8}, - - // 6D tensors (high-dim batch support) - {2, 1, 1, 4, 8, 16}, - {1, 1, 2, 3, 8, 8}, - }; - - // FP memory layouts to test - std::vector fp_layouts = { - utils::kWidthPacked, - utils::kChannelsPacked, - }; - - // Quantized memory layouts to test - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - // Test with buffer storage only - the unified block-based shaders only - // support buffer-backed floating-point tensors. Texture storage is tested - // separately by qdq8ta_conv2d_activations which uses the layout-specific - // shaders. - std::vector storage_types = { - utils::kBuffer, utils::kTexture3D}; - - // Generate all combinations - for (const auto& shape : shapes) { - // Generate test case name prefix from shape dimensions - std::string prefix = "ACCU"; - for (const auto& dim : shape) { - if (dim > kRefDimSizeLimit) { - prefix = "PERF"; - break; - } - } - - // Skip texture3d for high-dim tensors (textures are limited to 4D) - const bool is_highdim = shape.size() > 4; - const auto& effective_storage_types = is_highdim - ? std::vector{utils::kBuffer} - : storage_types; - - for (const auto& fp_layout : fp_layouts) { - for (const auto& quant_layout : quant_layouts) { - for (const auto& storage_type : effective_storage_types) { - QDQ8BitConfig config; - config.shape = shape; - config.test_case_name = prefix; - - test_cases.push_back(create_test_case_from_config( - config, storage_type, vkapi::kFloat, fp_layout, quant_layout)); - // For 4W4C layout, also test with legacy implementation - // (legacy path doesn't support high-dim tensors) - if (!is_highdim && fp_layout == utils::kChannelsPacked && - quant_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - storage_type, - vkapi::kFloat, - fp_layout, - quant_layout, - /*impl_selector=*/"legacy_4w4c")); - } - } - } - } - } - - return test_cases; -} - -// Reference implementation for q_dq_8bit operation -void q_dq_8bit_reference_impl(TestCase& test_case) { - int32_t idx = 0; - const ValueSpec& input_spec = test_case.inputs()[idx++]; - const ValueSpec& scale_spec = test_case.inputs()[idx++]; - const ValueSpec& zero_point_spec = test_case.inputs()[idx++]; - const ValueSpec& layout_spec = test_case.inputs()[idx++]; - (void)layout_spec; // Not used in reference implementation - - // Extract output specification - ValueSpec& output_spec = test_case.outputs()[0]; - - // Get tensor dimensions (arbitrary dimensionality) - auto input_sizes = input_spec.get_tensor_sizes(); - - // Calculate total number of elements - int64_t num_elements = 1; - for (const auto& dim : input_sizes) { - num_elements *= dim; - } - - // Skip for large tensors since computation time will be extremely slow - for (const auto& dim : input_sizes) { - if (dim > kRefDimSizeLimit) { - throw std::invalid_argument( - "One or more dimensions exceed the allowed limit for reference implementation."); - } - } - - if (input_spec.dtype != vkapi::kFloat) { - throw std::invalid_argument("Unsupported dtype"); - } - - // Get raw data pointers - auto& input_data = input_spec.get_float_data(); - - // Extract the randomized scale and zero point values - float scale = scale_spec.get_float_value(); - int32_t zero_point = zero_point_spec.get_int_value(); - int32_t quant_min = -128; - int32_t quant_max = 127; - - // Prepare output data - auto& ref_data = output_spec.get_ref_float_data(); - ref_data.resize(num_elements); - - // Perform quantize-dequantize operation on each element - for (int64_t i = 0; i < num_elements; ++i) { - float input_val = input_data[i]; - - // Quantize: quantized = round(input / scale + zero_point) - float quantized_float = std::round(input_val / scale) + zero_point; - - // Clamp to quantization range - quantized_float = std::max(quantized_float, static_cast(quant_min)); - quantized_float = std::min(quantized_float, static_cast(quant_max)); - - int32_t quantized_int = static_cast(quantized_float); - - // Dequantize: output = (quantized - zero_point) * scale - float dequantized = (quantized_int - zero_point) * scale; - - ref_data[i] = dequantized; - } -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(false); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Q/DQ 8-bit Per Tensor Operation Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = q_dq_8bit_reference_impl; - - auto results = execute_test_cases( -#ifdef DEBUG_MODE - generate_q_dq_8bit_easy_cases, -#else - generate_q_dq_8bit_test_cases, -#endif - "QDQ8Bit", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_sets/qdq_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_sets/qdq_sets.cpp new file mode 100644 index 00000000000..18aa2cce638 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_sets/qdq_sets.cpp @@ -0,0 +1,202 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_qdq { + +namespace { + +std::vector generate_test_cases( + const std::vector>& shapes, + const std::vector& fp_layouts, + const std::vector& quant_layouts, + const std::vector& storage_types, + const std::vector& legacy_4w4c_fp_layouts) { + std::vector test_cases; + + // Generate all combinations + for (const auto& shape : shapes) { + // Generate test case name prefix from shape dimensions + std::string prefix = "ACCU"; + for (const auto& dim : shape) { + if (dim > kRefDimSizeLimit) { + prefix = "PERF"; + break; + } + } + + const bool is_highdim = shape.size() > 4; + + for (const auto& fp_layout : fp_layouts) { + const bool test_legacy_4w4c = + std::find( + legacy_4w4c_fp_layouts.begin(), + legacy_4w4c_fp_layouts.end(), + fp_layout) != legacy_4w4c_fp_layouts.end(); + for (const auto& quant_layout : quant_layouts) { + for (const auto& storage_type : storage_types) { + // Textures are limited to 4D + if (is_highdim && storage_type != utils::kBuffer) { + continue; + } + QDQ8BitConfig config; + config.shape = shape; + config.test_case_name = prefix; + + test_cases.push_back(create_test_case_from_config( + config, storage_type, vkapi::kFloat, fp_layout, quant_layout)); + // For 4W4C layout, also test with legacy implementation + // (legacy path doesn't support high-dim tensors) + if (!is_highdim && test_legacy_4w4c && + quant_layout == utils::kPackedInt8_4W4C) { + test_cases.push_back(create_test_case_from_config( + config, + storage_type, + vkapi::kFloat, + fp_layout, + quant_layout, + /*impl_selector=*/"legacy_4w4c")); + } + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/qdq", "correctness") { + // Shapes to test (no layout specified - will be combined with all + // layouts) + const std::vector> shapes = { + // 1D tensors + {144}, + {90}, + + // 2D tensors (exercises block config with ndim < 4) + {1, 144}, + {1, 90}, + {1, 4}, + {3, 32}, + + // 3D tensors + {1, 16, 32}, + {1, 3, 64}, + + // Small test cases for correctness + {1, 3, 16, 16}, + {1, 8, 32, 32}, + {1, 16, 24, 24}, + {1, 32, 12, 12}, + {1, 1, 64, 64}, + {1, 3, 64, 64}, + {1, 4, 16, 16}, + + // Different tensor sizes + {1, 8, 20, 20}, + {1, 16, 14, 14}, + {1, 8, 28, 28}, + + // Odd tensor sizes + {1, 3, 15, 15}, + {1, 13, 31, 31}, + {1, 17, 23, 23}, + + // Performance test cases (larger tensors) + {1, 64, 128, 128}, + {1, 32, 64, 64}, + {1, 128, 56, 56}, + + // 5D tensors (high-dim batch support) + {2, 1, 3, 16, 16}, + {1, 2, 8, 8, 8}, + + // 6D tensors (high-dim batch support) + {2, 1, 1, 4, 8, 16}, + {1, 1, 2, 3, 8, 8}, + }; + // FP memory layouts to test + const std::vector fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + // Quantized memory layouts to test + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + // Test with buffer storage only - the unified block-based shaders only + // support buffer-backed floating-point tensors. Texture storage is tested + // separately by qdq8ta_conv2d_activations which uses the layout-specific + // shaders. + const std::vector storage_types = { + utils::kBuffer, utils::kTexture3D}; + const std::vector legacy_4w4c_fp_layouts = { + utils::kChannelsPacked}; + return { + generate_test_cases( + shapes, + fp_layouts, + quant_layouts, + storage_types, + legacy_4w4c_fp_layouts), + q_dq_8bit_reference_impl}; +} + +// Easy test cases for q_dq_8bit operation (for debugging) +REGISTER_TEST_CASE_SET("q8ta/qdq", "debug") { + const std::vector> shapes = { + {1, 16, 16, 16}, // 4D: [N, C, H, W] + {1, 144}, // 2D: exercises block config with ndim < 4 + {1, 90}, // 2D: matches skin_seg model's keypoint/bbox tensor sizes + {2, 1, 3, 16, 16}, // 5D: exercises high-dim batch support + {2, 1, 1, 4, 8, 16}, // 6D: exercises high-dim batch support + }; + // FP memory layouts to test + const std::vector fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + // Quantized memory layouts to test + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + const std::vector storage_types = {utils::kBuffer}; + const std::vector legacy_4w4c_fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + return { + generate_test_cases( + shapes, + fp_layouts, + quant_layouts, + storage_types, + legacy_4w4c_fp_layouts), + q_dq_8bit_reference_impl}; +} + +} // namespace q8ta_qdq +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/test_sets/unary_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/test_sets/unary_sets.cpp new file mode 100644 index 00000000000..2d7340ee916 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/test_sets/unary_sets.cpp @@ -0,0 +1,120 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_unary { + +namespace { + +std::vector generate_test_cases( + const std::vector>& shapes, + const std::vector& fp_layouts, + const std::vector& quant_layouts, + const std::vector& storage_types) { + std::vector test_cases; + + for (const auto& shape : shapes) { + std::string prefix = "ACCU"; + for (const auto& dim : shape) { + if (dim > kRefDimSizeLimit) { + prefix = "PERF"; + break; + } + } + + for (const auto& fp_layout : fp_layouts) { + for (const auto& quant_layout : quant_layouts) { + for (const auto& storage_type : storage_types) { + Q8taUnaryConfig config; + config.shape = shape; + config.test_case_name = prefix; + + test_cases.push_back(create_test_case_from_config( + config, storage_type, vkapi::kFloat, fp_layout, quant_layout)); + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/unary", "correctness") { + const std::vector> shapes = { + {1, 3, 16, 16}, + {1, 8, 32, 32}, + {1, 16, 24, 24}, + {1, 32, 12, 12}, + {1, 1, 64, 64}, + {1, 3, 64, 64}, + {1, 4, 16, 16}, + + {1, 8, 20, 20}, + {1, 16, 14, 14}, + {1, 8, 28, 28}, + + // Odd tensor sizes + {1, 3, 15, 15}, + {1, 13, 31, 31}, + {1, 17, 23, 23}, + + // Larger tensors + {1, 64, 128, 128}, + {1, 32, 64, 64}, + {1, 128, 56, 56}, + {1, 128, 128, 128}, + }; + const std::vector fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + const std::vector storage_types = {utils::kBuffer}; + return { + generate_test_cases(shapes, fp_layouts, quant_layouts, storage_types), + q8ta_unary_reference_impl}; +} + +REGISTER_TEST_CASE_SET("q8ta/unary", "debug") { + const std::vector> shapes = { + {1, 16, 16, 16}, + }; + const std::vector fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + const std::vector storage_types = {utils::kBuffer}; + return { + generate_test_cases(shapes, fp_layouts, quant_layouts, storage_types), + q8ta_unary_reference_impl}; +} + +} // namespace q8ta_unary +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/test_q8ta_unary.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/unary.cpp similarity index 55% rename from backends/vulkan/test/ops/benchmark/cases/q8ta/unary/test_q8ta_unary.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/unary/unary.cpp index 43eff9bc8ee..d11d5c41a79 100644 --- a/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/test_q8ta_unary.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/unary.cpp @@ -4,28 +4,17 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include -#include +#include + #include #include -#include -#include - -#include - -// #define DEBUG_MODE - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 512; +#include +#include -struct Q8taUnaryConfig { - std::vector shape; - std::string test_case_name = "placeholder"; - std::string op_name = "q8ta_unary_test"; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_unary { TestCase create_test_case_from_config( const Q8taUnaryConfig& config, @@ -101,113 +90,6 @@ TestCase create_test_case_from_config( return test_case; } -std::vector generate_q8ta_unary_easy_cases() { - std::vector test_cases; - - Q8taUnaryConfig config = { - {1, 16, 16, 16}, - "ACCU", - }; - - std::vector fp_layouts = { - utils::kWidthPacked, - utils::kChannelsPacked, - }; - - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - std::vector storage_types = {utils::kBuffer}; - std::vector float_types = {vkapi::kFloat}; - - for (const auto& fp_layout : fp_layouts) { - for (const auto& quant_layout : quant_layouts) { - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - test_cases.push_back(create_test_case_from_config( - config, storage_type, input_dtype, fp_layout, quant_layout)); - } - } - } - } - - return test_cases; -} - -std::vector generate_q8ta_unary_test_cases() { - std::vector test_cases; - - std::vector> shapes = { - {1, 3, 16, 16}, - {1, 8, 32, 32}, - {1, 16, 24, 24}, - {1, 32, 12, 12}, - {1, 1, 64, 64}, - {1, 3, 64, 64}, - {1, 4, 16, 16}, - - {1, 8, 20, 20}, - {1, 16, 14, 14}, - {1, 8, 28, 28}, - - // Odd tensor sizes - {1, 3, 15, 15}, - {1, 13, 31, 31}, - {1, 17, 23, 23}, - - // Larger tensors - {1, 64, 128, 128}, - {1, 32, 64, 64}, - {1, 128, 56, 56}, - {1, 128, 128, 128}, - }; - - std::vector fp_layouts = { - utils::kWidthPacked, - utils::kChannelsPacked, - }; - - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - std::vector storage_types = {utils::kBuffer}; - - for (const auto& shape : shapes) { - std::string prefix = "ACCU"; - for (const auto& dim : shape) { - if (dim > kRefDimSizeLimit) { - prefix = "PERF"; - break; - } - } - - for (const auto& fp_layout : fp_layouts) { - for (const auto& quant_layout : quant_layouts) { - for (const auto& storage_type : storage_types) { - Q8taUnaryConfig config; - config.shape = shape; - config.test_case_name = prefix; - - test_cases.push_back(create_test_case_from_config( - config, storage_type, vkapi::kFloat, fp_layout, quant_layout)); - } - } - } - } - - return test_cases; -} - // Reference implementation: quantize -> relu -> dequantize void q8ta_unary_reference_impl(TestCase& test_case) { int32_t idx = 0; @@ -280,32 +162,7 @@ void q8ta_unary_reference_impl(TestCase& test_case) { } } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(false); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Q8TA Unary (ReLU) Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = q8ta_unary_reference_impl; - - auto results = execute_test_cases( -#ifdef DEBUG_MODE - generate_q8ta_unary_easy_cases, -#else - generate_q8ta_unary_test_cases, -#endif - "Q8taUnary", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_unary +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/unary.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/unary.h new file mode 100644 index 00000000000..2ff17187a8e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/unary.h @@ -0,0 +1,40 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_unary { + +constexpr int64_t kRefDimSizeLimit = 512; + +struct Q8taUnaryConfig { + std::vector shape; + std::string test_case_name = "placeholder"; + std::string op_name = "q8ta_unary_test"; +}; + +TestCase create_test_case_from_config( + const Q8taUnaryConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype, + utils::GPUMemoryLayout fp_memory_layout, + utils::GPUMemoryLayout quant_layout); + +void q8ta_unary_reference_impl(TestCase& test_case); + +} // namespace q8ta_unary +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.cpp b/backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.cpp new file mode 100644 index 00000000000..24e39a3e110 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.cpp @@ -0,0 +1,202 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace sdpa { + +TestCase create_sdpa_test_case( + const SDPAConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + const std::string& impl) { + TestCase test_case; + + const bool is_perf = config.context_len > kRefContextLenLimit; + const std::string prefix = is_perf ? "PERF" : "ACCU"; + const std::string storage_str = repr_str(storage_type, utils::kWidthPacked); + const std::string dtype_str = dtype_short(dtype); + + const std::string shape = "D" + std::to_string(config.head_dim) + " H" + + std::to_string(config.n_heads) + " Hkv" + + std::to_string(config.n_kv_heads) + " S" + + std::to_string(config.seq_len) + " C" + + std::to_string(config.context_len); + + const std::string suffix = + "[" + config.model + " " + config.regime + " " + impl + "]"; + + test_case.set_name(make_test_label( + prefix, dtype_str, dtype_str, shape, storage_str, suffix)); + test_case.set_operator_name("test_etvk.test_sdpa.default"); + + // q: [1, S, n_heads, head_dim] + ValueSpec q( + {1, config.seq_len, config.n_heads, config.head_dim}, + dtype, + storage_type, + utils::kWidthPacked, + DataGenType::RANDOM); + + // k_cache / v_cache: [1, context_len, n_kv_heads, head_dim] + ValueSpec k_cache( + {1, config.context_len, config.n_kv_heads, config.head_dim}, + dtype, + storage_type, + utils::kWidthPacked, + DataGenType::RANDOM); + ValueSpec v_cache( + {1, config.context_len, config.n_kv_heads, config.head_dim}, + dtype, + storage_type, + utils::kWidthPacked, + DataGenType::RANDOM); + + ValueSpec impl_selector = ValueSpec::make_string(impl); + + // out: [1, S, n_heads, head_dim] + ValueSpec output( + {1, config.seq_len, config.n_heads, config.head_dim}, + dtype, + storage_type, + utils::kWidthPacked, + DataGenType::ZEROS); + + test_case.add_input_spec(q); + test_case.add_input_spec(k_cache); + test_case.add_input_spec(v_cache); + test_case.add_input_spec(impl_selector); + test_case.add_output_spec(output); + + if (dtype == vkapi::kHalf) { + test_case.set_abs_tolerance(1e-2f); + test_case.set_rel_tolerance(1e-2f); + } else { + test_case.set_abs_tolerance(1e-3f); + test_case.set_rel_tolerance(1e-3f); + } + + return test_case; +} + +// FLOPs: QK (2*S*C*D) + AV (2*S*C*D) per head, summed over heads. Softmax +// is negligible. Uses the causal-average context (~C/2) is ignored; report +// full-C dense FLOPs as an upper bound proxy. +int64_t sdpa_flop_calculator(const TestCase& test_case) { + const auto q_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto k_sizes = test_case.inputs()[1].get_tensor_sizes(); + const int64_t S = q_sizes[1]; + const int64_t H = q_sizes[2]; + const int64_t D = q_sizes[3]; + const int64_t C = k_sizes[1]; + return 4 * H * S * C * D; +} + +static std::vector as_float_data(const ValueSpec& spec) { + if (spec.dtype == vkapi::kFloat) { + return spec.get_float_data(); + } + if (spec.dtype == vkapi::kHalf) { + const auto& half_bits = spec.get_half_data(); + std::vector out(half_bits.size()); + for (size_t i = 0; i < half_bits.size(); ++i) { + out[i] = half_to_float(half_bits[i]); + } + return out; + } + throw std::invalid_argument("as_float_data: unsupported dtype"); +} + +// Reference: causal SDPA over the KV cache. +// q:[1,S,H,D], k/v cache:[1,C,Hkv,D], input_pos = C - S. +// For query row s (absolute position input_pos + s), attends to cache +// positions [0, input_pos + s]. GQA: head h maps to kv head h / (H/Hkv). +void sdpa_reference_impl(TestCase& test_case) { + const auto& q = test_case.inputs()[0]; + const auto& k = test_case.inputs()[1]; + const auto& v = test_case.inputs()[2]; + + const auto q_sizes = q.get_tensor_sizes(); + const auto k_sizes = k.get_tensor_sizes(); + + const int64_t S = q_sizes[1]; + const int64_t H = q_sizes[2]; + const int64_t D = q_sizes[3]; + const int64_t C = k_sizes[1]; + const int64_t Hkv = k_sizes[2]; + + if (C > kRefContextLenLimit) { + throw std::invalid_argument("sdpa reference: perf shape, skipping"); + } + + const int64_t input_pos = C - S; + const int64_t heads_per_kv = H / Hkv; + const float scale = 1.0f / std::sqrt(static_cast(D)); + + const auto q_data = as_float_data(q); + const auto k_data = as_float_data(k); + const auto v_data = as_float_data(v); + + ValueSpec& output = test_case.outputs()[0]; + auto& ref = output.get_ref_float_data(); + ref.assign(S * H * D, 0.0f); + + // Index helpers (contiguous WHCN-flattened as [1, dim1, dim2, dim3]). + auto q_idx = [&](int64_t s, int64_t h, int64_t d) { + return (s * H + h) * D + d; + }; + auto kv_idx = [&](int64_t c, int64_t hk, int64_t d) { + return (c * Hkv + hk) * D + d; + }; + + std::vector scores(C); + for (int64_t s = 0; s < S; ++s) { + const int64_t attend_len = input_pos + s + 1; // causal + for (int64_t h = 0; h < H; ++h) { + const int64_t hk = h / heads_per_kv; + + float max_score = -std::numeric_limits::infinity(); + for (int64_t c = 0; c < attend_len; ++c) { + float dot = 0.0f; + for (int64_t d = 0; d < D; ++d) { + dot += q_data[q_idx(s, h, d)] * k_data[kv_idx(c, hk, d)]; + } + dot *= scale; + scores[c] = dot; + max_score = std::max(max_score, dot); + } + + float denom = 0.0f; + for (int64_t c = 0; c < attend_len; ++c) { + scores[c] = std::exp(scores[c] - max_score); + denom += scores[c]; + } + + for (int64_t d = 0; d < D; ++d) { + float acc = 0.0f; + for (int64_t c = 0; c < attend_len; ++c) { + acc += scores[c] * v_data[kv_idx(c, hk, d)]; + } + ref[q_idx(s, h, d)] = acc / denom; + } + } + } +} + +} // namespace sdpa +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.h b/backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.h new file mode 100644 index 00000000000..a04e2784704 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.h @@ -0,0 +1,50 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace sdpa { + +// Correctness is only checked for these small shapes; larger perf shapes throw +// std::invalid_argument from the reference (framework marks them SKIPPED) to +// avoid an O(H*S*context*D) CPU reference on the large perf matrix. +constexpr int64_t kRefContextLenLimit = 256; + +// LLM SDPA (llama.custom_sdpa) shape: +// q: [1, S, n_heads, head_dim] (DHSB, width-packed) +// k/v cache:[1, context_len, n_kv_heads, head_dim] +struct SDPAConfig { + int64_t head_dim; + int64_t n_heads; + int64_t n_kv_heads; + int64_t seq_len; // S: query tokens (1 for decode, >1 for prefill) + int64_t context_len; // total KV length (kv_len) + std::string model; // label only + std::string regime; // "decode" / "prefill", label only +}; + +TestCase create_sdpa_test_case( + const SDPAConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + const std::string& impl); + +void sdpa_reference_impl(TestCase& test_case); + +int64_t sdpa_flop_calculator(const TestCase& test_case); + +} // namespace sdpa +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/sdpa/test_sdpa.cpp b/backends/vulkan/test/ops/benchmark/cases/sdpa/test_sdpa.cpp deleted file mode 100644 index d71860c5178..00000000000 --- a/backends/vulkan/test/ops/benchmark/cases/sdpa/test_sdpa.cpp +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include -#include -#include -#include - -#include -#include - -#include - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -// Correctness is only checked for these small shapes; larger perf shapes throw -// std::invalid_argument from the reference (framework marks them SKIPPED) to -// avoid an O(H*S*context*D) CPU reference on the large perf matrix. -static constexpr int64_t kRefContextLenLimit = 256; - -// When true (env SDPA_NO_CHAIN=1), each graph.execute() dispatches the SDPA op -// exactly once (op_invocations_per_execute=1), disabling the framework's -// probe-then-scale chaining. Chaining stacks many back-to-back QK/softmax/AV -// triples in one command buffer; consecutive triples pipeline on the GPU so -// their timestamp windows overlap, which inflates/misattributes per-dispatch -// durations (notably the AV dispatch) at the large chained_dispatches factors -// picked for the cheap small-context decode cases. One-invocation-per-execute -// removes the inter-invocation overlap; stability comes from warmup + a large -// median-of-N instead. -static bool no_chain_mode() { - const char* v = std::getenv("SDPA_NO_CHAIN"); - return v != nullptr && v[0] == '1'; -} - -// When true (env SDPA_DECODE_ONLY=1), generate only the 9 decode cases. -static bool decode_only_mode() { - const char* v = std::getenv("SDPA_DECODE_ONLY"); - return v != nullptr && v[0] == '1'; -} - -// LLM SDPA (llama.custom_sdpa) shape: -// q: [1, S, n_heads, head_dim] (DHSB, width-packed) -// k/v cache:[1, context_len, n_kv_heads, head_dim] -struct SDPAConfig { - int64_t head_dim; - int64_t n_heads; - int64_t n_kv_heads; - int64_t seq_len; // S: query tokens (1 for decode, >1 for prefill) - int64_t context_len; // total KV length (kv_len) - std::string model; // label only - std::string regime; // "decode" / "prefill", label only -}; - -static std::vector as_float_data(const ValueSpec& spec) { - if (spec.dtype == vkapi::kFloat) { - return spec.get_float_data(); - } - if (spec.dtype == vkapi::kHalf) { - const auto& half_bits = spec.get_half_data(); - std::vector out(half_bits.size()); - for (size_t i = 0; i < half_bits.size(); ++i) { - out[i] = half_to_float(half_bits[i]); - } - return out; - } - throw std::invalid_argument("as_float_data: unsupported dtype"); -} - -static TestCase create_sdpa_test_case( - const SDPAConfig& config, - vkapi::ScalarType dtype, - utils::StorageType storage_type, - const std::string& impl) { - TestCase test_case; - - const bool is_perf = config.context_len > kRefContextLenLimit; - const std::string prefix = is_perf ? "PERF" : "ACCU"; - const std::string storage_str = repr_str(storage_type, utils::kWidthPacked); - const std::string dtype_str = dtype_short(dtype); - - const std::string shape = "D" + std::to_string(config.head_dim) + " H" + - std::to_string(config.n_heads) + " Hkv" + - std::to_string(config.n_kv_heads) + " S" + - std::to_string(config.seq_len) + " C" + - std::to_string(config.context_len); - - const std::string suffix = - "[" + config.model + " " + config.regime + " " + impl + "]"; - - test_case.set_name(make_test_label( - prefix, dtype_str, dtype_str, shape, storage_str, suffix)); - test_case.set_operator_name("test_etvk.test_sdpa.default"); - - // q: [1, S, n_heads, head_dim] - ValueSpec q( - {1, config.seq_len, config.n_heads, config.head_dim}, - dtype, - storage_type, - utils::kWidthPacked, - DataGenType::RANDOM); - - // k_cache / v_cache: [1, context_len, n_kv_heads, head_dim] - ValueSpec k_cache( - {1, config.context_len, config.n_kv_heads, config.head_dim}, - dtype, - storage_type, - utils::kWidthPacked, - DataGenType::RANDOM); - ValueSpec v_cache( - {1, config.context_len, config.n_kv_heads, config.head_dim}, - dtype, - storage_type, - utils::kWidthPacked, - DataGenType::RANDOM); - - ValueSpec impl_selector = ValueSpec::make_string(impl); - - // out: [1, S, n_heads, head_dim] - ValueSpec output( - {1, config.seq_len, config.n_heads, config.head_dim}, - dtype, - storage_type, - utils::kWidthPacked, - DataGenType::ZEROS); - - test_case.add_input_spec(q); - test_case.add_input_spec(k_cache); - test_case.add_input_spec(v_cache); - test_case.add_input_spec(impl_selector); - test_case.add_output_spec(output); - - if (no_chain_mode()) { - test_case.set_op_invocations_per_execute(1); - } - - if (dtype == vkapi::kHalf) { - test_case.set_abs_tolerance(1e-2f); - test_case.set_rel_tolerance(1e-2f); - } else { - test_case.set_abs_tolerance(1e-3f); - test_case.set_rel_tolerance(1e-3f); - } - - return test_case; -} - -// Reference: causal SDPA over the KV cache. -// q:[1,S,H,D], k/v cache:[1,C,Hkv,D], input_pos = C - S. -// For query row s (absolute position input_pos + s), attends to cache -// positions [0, input_pos + s]. GQA: head h maps to kv head h / (H/Hkv). -static void sdpa_reference_impl(TestCase& test_case) { - const auto& q = test_case.inputs()[0]; - const auto& k = test_case.inputs()[1]; - const auto& v = test_case.inputs()[2]; - - const auto q_sizes = q.get_tensor_sizes(); - const auto k_sizes = k.get_tensor_sizes(); - - const int64_t S = q_sizes[1]; - const int64_t H = q_sizes[2]; - const int64_t D = q_sizes[3]; - const int64_t C = k_sizes[1]; - const int64_t Hkv = k_sizes[2]; - - if (C > kRefContextLenLimit) { - throw std::invalid_argument("sdpa reference: perf shape, skipping"); - } - - const int64_t input_pos = C - S; - const int64_t heads_per_kv = H / Hkv; - const float scale = 1.0f / std::sqrt(static_cast(D)); - - const auto q_data = as_float_data(q); - const auto k_data = as_float_data(k); - const auto v_data = as_float_data(v); - - ValueSpec& output = test_case.outputs()[0]; - auto& ref = output.get_ref_float_data(); - ref.assign(S * H * D, 0.0f); - - // Index helpers (contiguous WHCN-flattened as [1, dim1, dim2, dim3]). - auto q_idx = [&](int64_t s, int64_t h, int64_t d) { - return (s * H + h) * D + d; - }; - auto kv_idx = [&](int64_t c, int64_t hk, int64_t d) { - return (c * Hkv + hk) * D + d; - }; - - std::vector scores(C); - for (int64_t s = 0; s < S; ++s) { - const int64_t attend_len = input_pos + s + 1; // causal - for (int64_t h = 0; h < H; ++h) { - const int64_t hk = h / heads_per_kv; - - float max_score = -std::numeric_limits::infinity(); - for (int64_t c = 0; c < attend_len; ++c) { - float dot = 0.0f; - for (int64_t d = 0; d < D; ++d) { - dot += q_data[q_idx(s, h, d)] * k_data[kv_idx(c, hk, d)]; - } - dot *= scale; - scores[c] = dot; - max_score = std::max(max_score, dot); - } - - float denom = 0.0f; - for (int64_t c = 0; c < attend_len; ++c) { - scores[c] = std::exp(scores[c] - max_score); - denom += scores[c]; - } - - for (int64_t d = 0; d < D; ++d) { - float acc = 0.0f; - for (int64_t c = 0; c < attend_len; ++c) { - acc += scores[c] * v_data[kv_idx(c, hk, d)]; - } - ref[q_idx(s, h, d)] = acc / denom; - } - } - } -} - -// FLOPs: QK (2*S*C*D) + AV (2*S*C*D) per head, summed over heads. Softmax -// is negligible. Uses the causal-average context (~C/2) is ignored; report -// full-C dense FLOPs as an upper bound proxy. -static int64_t sdpa_flop_calculator(const TestCase& test_case) { - const auto q_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto k_sizes = test_case.inputs()[1].get_tensor_sizes(); - const int64_t S = q_sizes[1]; - const int64_t H = q_sizes[2]; - const int64_t D = q_sizes[3]; - const int64_t C = k_sizes[1]; - return 4 * H * S * C * D; -} - -static std::vector generate_sdpa_test_cases() { - std::vector test_cases; - - struct ModelDims { - std::string name; - int64_t head_dim; - int64_t n_heads; - int64_t n_kv_heads; - }; - - const std::vector models = { - {"Llama-3.2-1B", 64, 32, 8}, - {"Qwen3-0.6B", 128, 16, 8}, - {"Phi-4-mini", 128, 24, 8}, - }; - - // Decode: S=1, sweep context_len. - const std::vector decode_context_lens = {512, 1024, 4096}; - // Prefill: S == context_len. - const std::vector prefill_seq_lens = {128, 512}; - - // Perf runs use fp16 texture (matches the LLM decode/prefill production - // path). A couple of small ACCU shapes validate correctness in fp32. - const auto dtype = vkapi::kHalf; - const auto storage = utils::kTexture3D; - - // Decode (S==1) picks a coop AV shader; exercise both the GQA-reuse variant - // and the per-query-head variant for every decode case. Prefill (tiled) is - // unaffected by the selector, so it runs a single case. - const std::vector decode_impls = {"gqa", "non_gqa"}; - - for (const auto& m : models) { - for (int64_t c : decode_context_lens) { - SDPAConfig cfg; - cfg.head_dim = m.head_dim; - cfg.n_heads = m.n_heads; - cfg.n_kv_heads = m.n_kv_heads; - cfg.seq_len = 1; - cfg.context_len = c; - cfg.model = m.name; - cfg.regime = "decode"; - for (const auto& impl : decode_impls) { - test_cases.push_back(create_sdpa_test_case(cfg, dtype, storage, impl)); - } - } - if (decode_only_mode()) { - continue; - } - for (int64_t s : prefill_seq_lens) { - SDPAConfig cfg; - cfg.head_dim = m.head_dim; - cfg.n_heads = m.n_heads; - cfg.n_kv_heads = m.n_kv_heads; - cfg.seq_len = s; - cfg.context_len = s; - cfg.model = m.name; - cfg.regime = "prefill"; - test_cases.push_back( - create_sdpa_test_case(cfg, dtype, storage, "default")); - } - } - - if (decode_only_mode()) { - return test_cases; - } - - // Small ACCU correctness cases (fp32), decode + prefill. Texture is the - // production LLM path; buffer is also validated for decode to guard the - // attn_weights S/context alignment (a decode-shaped buffer allocation has no - // headroom for the shaders' align_up_4 stride unless padded — see sdpa_impl). - { - // Cover D=64 (D4=16) and D=128 (D4=32) with the vendor-default GQA and the - // per-query-head shaders, across texture + buffer. - const std::vector decs = { - {64, 8, 2, 1, 32, "accu", "decode"}, - {128, 8, 2, 1, 32, "accu_d128", "decode"}, - }; - for (const auto& dec : decs) { - for (const auto& storage : {utils::kTexture3D, utils::kBuffer}) { - for (const auto& impl : decode_impls) { - test_cases.push_back( - create_sdpa_test_case(dec, vkapi::kFloat, storage, impl)); - } - } - } - - // Force the head_dim output-tiled GQA variant (Adreno-only in production) - // so its wg x-collapse and the partial_n_tile tail get deterministic - // coverage on any device: D=64/128 give even D4 (fast path); D=4 gives D4=1 - // (odd), exercising the partial-tile checked load. - const std::vector tile2_decs = { - {64, 8, 2, 1, 32, "accu_tile2", "decode"}, - {128, 8, 2, 1, 32, "accu_tile2_d128", "decode"}, - {4, 8, 2, 1, 32, "accu_tile2_d4", "decode"}, - }; - for (const auto& dec : tile2_decs) { - for (const auto& storage : {utils::kTexture3D, utils::kBuffer}) { - test_cases.push_back( - create_sdpa_test_case(dec, vkapi::kFloat, storage, "gqa_tile2")); - } - } - - SDPAConfig pre{64, 8, 2, 16, 16, "accu", "prefill"}; - test_cases.push_back(create_sdpa_test_case( - pre, vkapi::kFloat, utils::kTexture3D, "default")); - } - - return test_cases; -} - -int main(int argc, char* argv[]) { - (void)argc; - (void)argv; - - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "SDPA (llama.custom_sdpa) Benchmark" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = sdpa_reference_impl; - - const bool decode_only = decode_only_mode(); - const int warmup_runs = decode_only ? 10 : 3; - const int benchmark_runs = decode_only ? 30 : 10; - - auto results = execute_test_cases( - generate_sdpa_test_cases, - sdpa_flop_calculator, - "SDPA", - warmup_runs, - benchmark_runs, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/ops/benchmark/cases/sdpa/test_sets/sdpa_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/sdpa/test_sets/sdpa_sets.cpp new file mode 100644 index 00000000000..95b1b341713 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/sdpa/test_sets/sdpa_sets.cpp @@ -0,0 +1,180 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace sdpa { + +namespace { + +struct ModelDims { + std::string name; + int64_t head_dim; + int64_t n_heads; + int64_t n_kv_heads; +}; + +const std::vector kModels = { + {"Llama-3.2-1B", 64, 32, 8}, + {"Qwen3-0.6B", 128, 16, 8}, + {"Phi-4-mini", 128, 24, 8}, +}; + +// fp32 cases: each config yields one case per storage type x impl. +struct SDPACaseGroup { + std::vector configs; + std::vector storage_types; + std::vector impls; +}; + +struct SDPASetOptions { + // Swept over every model in kModels (fp16, texture). Each decode context + // length yields an S=1 case per decode impl; each prefill sequence length + // yields one S == context_len case. + std::vector decode_context_lens = {}; + std::vector decode_impls = {}; + std::vector prefill_seq_lens = {}; + // Appended after the model sweep. + std::vector accu_cases = {}; +}; + +std::vector generate_test_cases(const SDPASetOptions& options) { + std::vector test_cases; + + // Perf runs use fp16 texture (matches the LLM decode/prefill production + // path). A couple of small ACCU shapes validate correctness in fp32. + const auto dtype = vkapi::kHalf; + const auto storage = utils::kTexture3D; + + for (const auto& m : kModels) { + for (int64_t c : options.decode_context_lens) { + SDPAConfig cfg; + cfg.head_dim = m.head_dim; + cfg.n_heads = m.n_heads; + cfg.n_kv_heads = m.n_kv_heads; + cfg.seq_len = 1; + cfg.context_len = c; + cfg.model = m.name; + cfg.regime = "decode"; + for (const auto& impl : options.decode_impls) { + test_cases.push_back(create_sdpa_test_case(cfg, dtype, storage, impl)); + } + } + for (int64_t s : options.prefill_seq_lens) { + SDPAConfig cfg; + cfg.head_dim = m.head_dim; + cfg.n_heads = m.n_heads; + cfg.n_kv_heads = m.n_kv_heads; + cfg.seq_len = s; + cfg.context_len = s; + cfg.model = m.name; + cfg.regime = "prefill"; + // Prefill (tiled) is unaffected by the impl selector, so it runs a + // single case. + test_cases.push_back( + create_sdpa_test_case(cfg, dtype, storage, "default")); + } + } + + for (const auto& group : options.accu_cases) { + for (const auto& cfg : group.configs) { + for (const auto& st : group.storage_types) { + for (const auto& impl : group.impls) { + test_cases.push_back( + create_sdpa_test_case(cfg, vkapi::kFloat, st, impl)); + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("sdpa", "correctness") { + return { + generate_test_cases({ + // Prefill: S == context_len. + .prefill_seq_lens = {128}, + // Small ACCU correctness cases (fp32), decode + prefill. Texture is + // the production LLM path; buffer is also validated for decode to + // guard the attn_weights S/context alignment (a decode-shaped buffer + // allocation has no headroom for the shaders' align_up_4 stride + // unless padded — see sdpa_impl). + .accu_cases = + { + // Cover D=64 (D4=16) and D=128 (D4=32) with the + // vendor-default GQA and the per-query-head shaders, across + // texture + buffer. + {.configs = + { + {64, 8, 2, 1, 32, "accu", "decode"}, + {128, 8, 2, 1, 32, "accu_d128", "decode"}, + }, + .storage_types = {utils::kTexture3D, utils::kBuffer}, + .impls = {"gqa", "non_gqa"}}, + // Force the head_dim output-tiled GQA variant (Adreno-only + // in production) so its wg x-collapse and the + // partial_n_tile tail get deterministic coverage on any + // device: D=64/128 give even D4 (fast path); D=4 gives D4=1 + // (odd), exercising the partial-tile checked load. + {.configs = + { + {64, 8, 2, 1, 32, "accu_tile2", "decode"}, + {128, 8, 2, 1, 32, "accu_tile2_d128", "decode"}, + {4, 8, 2, 1, 32, "accu_tile2_d4", "decode"}, + }, + .storage_types = {utils::kTexture3D, utils::kBuffer}, + .impls = {"gqa_tile2"}}, + {.configs = {{64, 8, 2, 16, 16, "accu", "prefill"}}, + .storage_types = {utils::kTexture3D}, + .impls = {"default"}}, + }, + }), + sdpa_reference_impl, + sdpa_flop_calculator, + /*warmup_runs=*/3, + /*benchmark_runs=*/10}; +} + +REGISTER_TEST_CASE_SET("sdpa", "decode") { + return { + generate_test_cases({ + // Decode: S=1, sweep context_len. + .decode_context_lens = {512, 1024, 4096}, + // Decode (S==1) picks a coop AV shader; exercise both the GQA-reuse + // variant and the per-query-head variant for every decode case. + .decode_impls = {"gqa", "non_gqa"}, + }), + sdpa_reference_impl, + sdpa_flop_calculator, + /*warmup_runs=*/10, + /*benchmark_runs=*/30}; +} + +REGISTER_TEST_CASE_SET("sdpa", "performance") { + return { + generate_test_cases({ + // Prefill: S == context_len. + .prefill_seq_lens = {512}, + }), + sdpa_reference_impl, + sdpa_flop_calculator, + /*warmup_runs=*/3, + /*benchmark_runs=*/10}; +} + +} // namespace sdpa +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/etvk_bench.cpp b/backends/vulkan/test/ops/benchmark/etvk_bench.cpp new file mode 100644 index 00000000000..9b642d8dc2d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/etvk_bench.cpp @@ -0,0 +1,247 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +// Runs the test case sets registered under cases/. See --helpshort for usage. + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +DEFINE_bool(list, false, "List the selected sets as : and exit."); +DEFINE_string( + op, + "", + "Comma-separated operators whose sets to run. An operator is its " + "directory under cases/; a parent directory such as q8ta selects every " + "operator beneath it. Empty selects all operators."); +DEFINE_string( + set, + "", + "Comma-separated set names to run. Empty selects all sets."); +DEFINE_string( + filter, + "", + "Run only test cases whose name matches the filter: '|'-separated " + "alternatives, each a ';'-separated list of substrings that must all " + "appear. For example, 'f16;+bias|f32' selects f16 cases with bias and all " + "f32 cases."); +DEFINE_int32( + chain, + 0, + "Dispatch the operator this many times per measured execute. 0 sizes the " + "chain adaptively."); +DEFINE_bool( + gpu_timestamps, + true, + "Time with GPU timestamps. --nogpu_timestamps uses the wall clock."); +DEFINE_bool(print_output, false, "Print output tensor data."); +DEFINE_bool(print_latencies, false, "Print per-iteration latencies."); +DEFINE_bool(debug, false, "Print debugging information."); +DEFINE_bool(device_info, false, "Print the Vulkan device properties and exit."); + +using namespace executorch::vulkan::prototyping; + +namespace { + +std::vector split(const std::string& value, char delimiter) { + std::vector items; + std::stringstream stream(value); + std::string item; + while (std::getline(stream, item, delimiter)) { + if (!item.empty()) { + items.push_back(item); + } + } + return items; +} + +bool op_selected(const std::vector& ops, const std::string& op) { + if (ops.empty()) { + return true; + } + for (const std::string& sel : ops) { + if (op == sel || op.rfind(sel + "/", 0) == 0) { + return true; + } + } + return false; +} + +bool set_selected( + const std::vector& sets, + const std::string& name) { + return sets.empty() || + std::find(sets.begin(), sets.end(), name) != sets.end(); +} + +// One list of required substrings per alternative. +using NameFilter = std::vector>; + +NameFilter parse_filter(const std::string& value) { + NameFilter filter; + for (const std::string& alternative : split(value, '|')) { + std::vector substrings = split(alternative, ';'); + if (!substrings.empty()) { + filter.push_back(std::move(substrings)); + } + } + return filter; +} + +bool contains_all( + const std::string& name, + const std::vector& substrings) { + for (const std::string& substring : substrings) { + if (name.find(substring) == std::string::npos) { + return false; + } + } + return true; +} + +bool name_matches(const NameFilter& filter, const std::string& name) { + if (filter.empty()) { + return true; + } + for (const std::vector& substrings : filter) { + if (contains_all(name, substrings)) { + return true; + } + } + return false; +} + +bool set_precedes( + const RegisteredTestCaseSet* a, + const RegisteredTestCaseSet* b) { + return a->op != b->op ? a->op < b->op : a->name < b->name; +} + +std::vector sorted_sets() { + std::vector sets; + for (const RegisteredTestCaseSet& set : registered_test_case_sets()) { + sets.push_back(&set); + } + std::sort(sets.begin(), sets.end(), set_precedes); + return sets; +} + +// Returns true if the set passed. +bool run_set( + const RegisteredTestCaseSet& registered, + const NameFilter& filter) { + const std::string id = registered.op + ":" + registered.name; + print_performance_header(); + std::cout << id << std::endl; + print_separator(); + + try { + TestCaseSet set = registered.make(); + if (set.custom_test) { + set.custom_test(); + return true; + } + std::vector test_cases; + for (TestCase& test_case : set.test_cases) { + if (!name_matches(filter, test_case.name())) { + continue; + } + if (FLAGS_chain > 0) { + test_case.set_op_invocations_per_execute(FLAGS_chain); + } + test_cases.push_back(std::move(test_case)); + } + execute_test_cases( + std::move(test_cases), + set.flop_calculator, + id, + set.warmup_runs, + set.benchmark_runs, + set.reference); + } catch (const std::exception& e) { + std::cerr << id << " failed: " << e.what() << std::endl; + return false; + } + return true; +} + +} // namespace + +int main(int argc, char* argv[]) { + gflags::SetUsageMessage( + "Runs the test case sets registered under cases/. With no selection " + "flags, every set runs."); + gflags::ParseCommandLineFlags(&argc, &argv, true); + if (argc > 1) { + std::cerr << "Unexpected argument: " << argv[1] << std::endl; + return 2; + } + if (FLAGS_chain < 0) { + std::cerr << "--chain must not be negative" << std::endl; + return 2; + } + + if (FLAGS_device_info) { + print_device_info(); + return 0; + } + + const std::vector ops = split(FLAGS_op, ','); + const std::vector set_names = split(FLAGS_set, ','); + std::vector selected; + for (const RegisteredTestCaseSet* set : sorted_sets()) { + if (op_selected(ops, set->op) && set_selected(set_names, set->name)) { + selected.push_back(set); + } + } + if (selected.empty()) { + std::cerr << "No test case sets match the selection" << std::endl; + return 2; + } + + if (FLAGS_list) { + for (const RegisteredTestCaseSet* set : selected) { + std::cout << set->op << ":" << set->name << std::endl; + } + return 0; + } + + set_debugging(FLAGS_debug); + set_print_output(FLAGS_print_output); + set_print_latencies(FLAGS_print_latencies); + set_use_gpu_timestamps(FLAGS_gpu_timestamps); + + const NameFilter filter = parse_filter(FLAGS_filter); + std::vector failed; + for (const RegisteredTestCaseSet* set : selected) { + if (!run_set(*set, filter)) { + failed.push_back(set->op + ":" + set->name); + } + } + + print_separator(); + std::cout << "Ran " << selected.size() << " test case sets"; + if (failed.empty()) { + std::cout << ", all passed" << std::endl; + } else { + std::cout << ", " << failed.size() << " failed" << std::endl; + } + for (const std::string& id : failed) { + std::cout << " FAILED: " << id << std::endl; + } + return failed.empty() ? 0 : 1; +} diff --git a/backends/vulkan/test/ops/benchmark/framework/test_coopmat_probe.cpp b/backends/vulkan/test/ops/benchmark/framework/device_info.cpp similarity index 77% rename from backends/vulkan/test/ops/benchmark/framework/test_coopmat_probe.cpp rename to backends/vulkan/test/ops/benchmark/framework/device_info.cpp index 257a5df95fc..5275b7c6121 100644 --- a/backends/vulkan/test/ops/benchmark/framework/test_coopmat_probe.cpp +++ b/backends/vulkan/test/ops/benchmark/framework/device_info.cpp @@ -4,20 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -// One-shot capability probe: prints the cooperative matrix configurations -// exposed by the active Vulkan device, plus the relevant adapter properties -// (subgroup size, iGPU vs dGPU, device name). Used to gate the coopmat -// implementations for quantized linear shaders. +#include "device_info.h" +#include "cm_utils.h" #include -#include "cm_utils.h" - #include -using namespace executorch::vulkan::prototyping; +namespace executorch { +namespace vulkan { +namespace prototyping { -int main() { +void print_device_info() { auto* adapter = vkcompute::api::context()->adapter_ptr(); std::cout << "=== Vulkan adapter ===\n"; @@ -36,6 +34,8 @@ int main() { << (adapter->supports_int8_dot_product() ? "yes" : "no") << "\n"; queryCooperativeMatrixProperties(); - - return 0; } + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/device_info.h b/backends/vulkan/test/ops/benchmark/framework/device_info.h new file mode 100644 index 00000000000..9094ca8d5cb --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/device_info.h @@ -0,0 +1,19 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// Prints the active Vulkan adapter's properties and the cooperative matrix +// configurations it exposes. +void print_device_info(); + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/registry.cpp b/backends/vulkan/test/ops/benchmark/framework/registry.cpp new file mode 100644 index 00000000000..da37f972e70 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/registry.cpp @@ -0,0 +1,48 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "registry.h" + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +namespace { + +std::vector& registry() { + static std::vector sets; + return sets; +} + +} // namespace + +const std::vector& registered_test_case_sets() { + return registry(); +} + +TestCaseSetRegistrar::TestCaseSetRegistrar( + std::string op, + std::string name, + std::function make) { + for (const RegisteredTestCaseSet& set : registry()) { + VK_CHECK_COND( + set.op != op || set.name != name, + "Test case set ", + op, + ":", + name, + " is registered more than once"); + } + registry().push_back({std::move(op), std::move(name), std::move(make)}); +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/registry.h b/backends/vulkan/test/ops/benchmark/framework/registry.h new file mode 100644 index 00000000000..ca24db5f47c --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/registry.h @@ -0,0 +1,67 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include "runner.h" +#include "test_case.h" + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +struct TestCaseSet { + std::vector test_cases; + ReferenceComputeFunc reference = nullptr; + FlopCalculatorFunc flop_calculator = default_flop_calculator; + int warmup_runs = 1; + int benchmark_runs = 1; + // Runs instead of test_cases for checks that are not expressible as test + // cases. Throws on failure. + std::function custom_test = nullptr; +}; + +struct RegisteredTestCaseSet { + // Path of the operator's directory under cases/, e.g. "q8ta/conv2d". + std::string op; + std::string name; + // Invoked only when the set is selected, since building test cases may + // query the device. + std::function make; +}; + +const std::vector& registered_test_case_sets(); + +class TestCaseSetRegistrar final { + public: + TestCaseSetRegistrar( + std::string op, + std::string name, + std::function make); +}; + +#define ETVK_TEST_CASE_SET_CONCAT_IMPL(a, b) a##b +#define ETVK_TEST_CASE_SET_CONCAT(a, b) ETVK_TEST_CASE_SET_CONCAT_IMPL(a, b) + +// Registers the TestCaseSet returned by the function body that follows. +#define REGISTER_TEST_CASE_SET(op, name) \ + static ::executorch::vulkan::prototyping::TestCaseSet \ + ETVK_TEST_CASE_SET_CONCAT(make_test_case_set_, __LINE__)(); \ + static const ::executorch::vulkan::prototyping::TestCaseSetRegistrar \ + ETVK_TEST_CASE_SET_CONCAT(test_case_set_registrar_, __LINE__)( \ + op, \ + name, \ + &ETVK_TEST_CASE_SET_CONCAT(make_test_case_set_, __LINE__)); \ + static ::executorch::vulkan::prototyping::TestCaseSet \ + ETVK_TEST_CASE_SET_CONCAT(make_test_case_set_, __LINE__)() + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/runner.cpp b/backends/vulkan/test/ops/benchmark/framework/runner.cpp index 3d5ce763bb5..35c030da58a 100644 --- a/backends/vulkan/test/ops/benchmark/framework/runner.cpp +++ b/backends/vulkan/test/ops/benchmark/framework/runner.cpp @@ -480,7 +480,7 @@ BenchmarkResult execute_test_case( } TestResult execute_test_cases( - const std::function()>& test_case_generator, + std::vector test_cases, const FlopCalculatorFunc& flop_calculator, const std::string& operation_name, int warmup_runs, @@ -488,9 +488,6 @@ TestResult execute_test_cases( const ReferenceComputeFunc& reference_compute_func) { TestResult results(operation_name); - // Generate all test cases - std::vector test_cases = test_case_generator(); - std::cout << "Executing " << test_cases.size() << " test cases for " << operation_name << std::endl; print_separator(); @@ -741,13 +738,13 @@ TestResult execute_test_cases( // Convenience overload that uses the default FLOP calculator TestResult execute_test_cases( - const std::function()>& test_case_generator, + std::vector test_cases, const std::string& operation_name, int warmup_runs, int benchmark_runs, const ReferenceComputeFunc& reference_compute_func) { return execute_test_cases( - test_case_generator, + std::move(test_cases), default_flop_calculator, operation_name, warmup_runs, diff --git a/backends/vulkan/test/ops/benchmark/framework/runner.h b/backends/vulkan/test/ops/benchmark/framework/runner.h index b3493aed9cb..56a7462b3c6 100644 --- a/backends/vulkan/test/ops/benchmark/framework/runner.h +++ b/backends/vulkan/test/ops/benchmark/framework/runner.h @@ -95,7 +95,7 @@ BenchmarkResult execute_test_case( bool write_outputs = true); TestResult execute_test_cases( - const std::function()>& test_case_generator, + std::vector test_cases, const FlopCalculatorFunc& flop_calculator, const std::string& operation_name = "Operation", int warmup_runs = 1, @@ -103,7 +103,7 @@ TestResult execute_test_cases( const ReferenceComputeFunc& reference_compute_func = nullptr); TestResult execute_test_cases( - const std::function()>& test_case_generator, + std::vector test_cases, const std::string& operation_name = "Operation", int warmup_runs = 1, int benchmark_runs = 1, diff --git a/backends/vulkan/test/ops/benchmark/targets.bzl b/backends/vulkan/test/ops/benchmark/targets.bzl index ec70bb549f1..d53f252cc65 100644 --- a/backends/vulkan/test/ops/benchmark/targets.bzl +++ b/backends/vulkan/test/ops/benchmark/targets.bzl @@ -5,24 +5,6 @@ load( "vulkan_spv_shader_lib", ) -def define_custom_op_test_binary(custom_op_name, subdir, extra_deps = [], include_torch = False): - deps_list = [ - ":prototyping_utils", - ":operator_implementations", - ":custom_ops_shaderlib", - "//executorch/backends/vulkan:vulkan_graph_runtime", - ] + ([runtime.external_dep_location("libtorch")] if include_torch else []) + extra_deps - - runtime.cxx_binary( - name = custom_op_name, - srcs = [ - "{}/{}.cpp".format(subdir, custom_op_name), - ], - platforms = get_platforms(), - define_static_target = False, - deps = deps_list, - ) - def define_common_targets(is_fbcode = False): if is_fbcode: return @@ -48,8 +30,11 @@ def define_common_targets(is_fbcode = False): runtime.cxx_library( name = "prototyping_utils", srcs = [ + "framework/cm_utils.cpp", "framework/config.cpp", "framework/conv2d_utils.cpp", + "framework/device_info.cpp", + "framework/registry.cpp", "framework/results.cpp", "framework/runner.cpp", "framework/test_case.cpp", @@ -57,9 +42,12 @@ def define_common_targets(is_fbcode = False): "framework/weight_utils.cpp", ], headers = [ + "framework/cm_utils.h", "framework/config.h", "framework/conv2d_utils.h", + "framework/device_info.h", "framework/labels.h", + "framework/registry.h", "framework/results.h", "framework/runner.h", "framework/test_case.h", @@ -68,9 +56,12 @@ def define_common_targets(is_fbcode = False): "framework/weight_utils.h", ], exported_headers = [ + "framework/cm_utils.h", "framework/config.h", "framework/conv2d_utils.h", + "framework/device_info.h", "framework/labels.h", + "framework/registry.h", "framework/results.h", "framework/runner.h", "framework/test_case.h", @@ -100,30 +91,19 @@ def define_common_targets(is_fbcode = False): link_whole = True, ) - - - - define_custom_op_test_binary("test_add", "cases/add") - define_custom_op_test_binary("test_q8csw_linear", "cases/q8csw_linear") - define_custom_op_test_binary("test_q8csw_conv2d", "cases/q8csw_conv2d") - define_custom_op_test_binary("test_choose_qparams_per_row", "cases/choose_qparams_per_row") - define_custom_op_test_binary("test_q4gsw_linear", "cases/q4gsw_linear") - define_custom_op_test_binary("test_q8ta_qdq", "cases/q8ta/qdq") - define_custom_op_test_binary("test_q8ta_clone", "cases/q8ta/clone") - define_custom_op_test_binary("test_q8ta_binary", "cases/q8ta/binary") - define_custom_op_test_binary("test_q8ta_conv2d", "cases/q8ta/conv2d") - define_custom_op_test_binary("test_q8ta_conv2d_pw", "cases/q8ta/conv2d") - define_custom_op_test_binary("test_q8ta_conv2d_dw", "cases/q8ta/conv2d") - define_custom_op_test_binary("test_q8ta_linear", "cases/q8ta/linear") - define_custom_op_test_binary("test_q8ta_conv2d_transposed", "cases/q8ta/conv2d_transposed") - define_custom_op_test_binary("test_q8ta_pixel_shuffle", "cases/q8ta/pixel_shuffle") - define_custom_op_test_binary("test_q8ta_unary", "cases/q8ta/unary") - define_custom_op_test_binary("test_mm", "cases/mm") - define_custom_op_test_binary("test_conv2d", "cases/conv2d") - define_custom_op_test_binary("test_conv2d_pw", "cases/conv2d") - define_custom_op_test_binary("test_conv2d_dw", "cases/conv2d") - define_custom_op_test_binary("test_embedding_q4gsw", "cases/embedding_q4gsw") - define_custom_op_test_binary("test_conv1d_pw", "cases/conv1d") - define_custom_op_test_binary("test_conv1d_dw", "cases/conv1d") - define_custom_op_test_binary("test_fpa_q4gsw_linear", "cases/q4gsw_linear") - define_custom_op_test_binary("test_sdpa", "cases/sdpa") + runtime.cxx_binary( + name = "etvk_bench", + srcs = ["etvk_bench.cpp"] + native.glob(["cases/**/*.cpp"]), + headers = native.glob(["cases/**/*.h"]), + platforms = get_platforms(), + define_static_target = False, + deps = [ + ":custom_ops_shaderlib", + ":operator_implementations", + ":prototyping_utils", + "//executorch/backends/vulkan:vulkan_graph_runtime", + ], + external_deps = [ + "gflags", + ], + )