diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 97ca6969318..6e5b0bc46a1 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -1680,14 +1680,15 @@ jobs: .ci/scripts/setup-linux.sh --build-tool "cmake" # Custom operator tests - PYTHON_EXECUTABLE=python bash backends/vulkan/test/custom_ops/build_and_run.sh test_add - ./cmake-out/backends/vulkan/test/custom_ops/test_q8csw_linear - ./cmake-out/backends/vulkan/test/custom_ops/test_q8csw_conv2d - ./cmake-out/backends/vulkan/test/custom_ops/test_q4gsw_linear - ./cmake-out/backends/vulkan/test/custom_ops/test_choose_qparams_per_row - ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_qdq - ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_clone - ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_binary + PYTHON_EXECUTABLE=python bash backends/vulkan/test/ops/benchmark/build_and_run.sh + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=add + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8csw_linear --set=correctness,performance + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8csw_conv2d + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q4gsw_linear --set=correctness,performance + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=choose_qparams_per_row --set=correctness,performance + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8ta/qdq --set=correctness + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8ta/clone --set=correctness + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8ta/binary --set=correctness # "Classic" Operator tests PYTHON_EXECUTABLE=python bash backends/vulkan/test/scripts/test_op.sh --build diff --git a/.github/workflows/vulkan.yml b/.github/workflows/vulkan.yml index 223fadebf23..9397707d979 100644 --- a/.github/workflows/vulkan.yml +++ b/.github/workflows/vulkan.yml @@ -98,17 +98,17 @@ jobs: done # Operator coverage (mirrors test-vulkan-operators-linux, on real hardware). - # The custom-op prototyping binaries are GPU microbenchmarks that rely on - # GPU timestamp queries, so they require the hardware selected above. - PYTHON_EXECUTABLE=python bash backends/vulkan/test/custom_ops/build_and_run.sh - ./cmake-out/backends/vulkan/test/custom_ops/test_add - ./cmake-out/backends/vulkan/test/custom_ops/test_q8csw_linear - ./cmake-out/backends/vulkan/test/custom_ops/test_q8csw_conv2d - ./cmake-out/backends/vulkan/test/custom_ops/test_q4gsw_linear - ./cmake-out/backends/vulkan/test/custom_ops/test_choose_qparams_per_row - ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_qdq - ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_clone - ./cmake-out/backends/vulkan/test/custom_ops/test_q8ta_binary + # etvk_bench runs GPU microbenchmarks that rely on GPU timestamp queries, + # so it requires the hardware selected above. + PYTHON_EXECUTABLE=python bash backends/vulkan/test/ops/benchmark/build_and_run.sh + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=add + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8csw_linear --set=correctness,performance + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8csw_conv2d + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q4gsw_linear --set=correctness,performance + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=choose_qparams_per_row --set=correctness,performance + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8ta/qdq --set=correctness + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8ta/clone --set=correctness + ./cmake-out/backends/vulkan/test/ops/benchmark/etvk_bench --op=q8ta/binary --set=correctness PYTHON_EXECUTABLE=python bash backends/vulkan/test/scripts/test_op.sh --build vulkan_sdpa_av_test diff --git a/backends/vulkan/cmake/ShaderLibrary.cmake b/backends/vulkan/cmake/ShaderLibrary.cmake index d7e55cd3592..53d2839a45e 100644 --- a/backends/vulkan/cmake/ShaderLibrary.cmake +++ b/backends/vulkan/cmake/ShaderLibrary.cmake @@ -81,8 +81,14 @@ function(gen_vulkan_shader_lib_cpp shaders_path) endif() # Ninja cannot expand wildcards (*) in DEPENDS lists. - file(GLOB VULKAN_SHADERS "${shaders_path}/*.glsl" "${shaders_path}/*.glslh" - "${shaders_path}/*.yaml" "${shaders_path}/*.h" + file( + GLOB_RECURSE + VULKAN_SHADERS + CONFIGURE_DEPENDS + "${shaders_path}/*.glsl" + "${shaders_path}/*.glslh" + "${shaders_path}/*.yaml" + "${shaders_path}/*.h" ) add_custom_command( diff --git a/backends/vulkan/op_registry.py b/backends/vulkan/op_registry.py index 6f4cd9b8978..019c141b6fc 100644 --- a/backends/vulkan/op_registry.py +++ b/backends/vulkan/op_registry.py @@ -248,7 +248,7 @@ def register_clamp(): # ============================================================================= -# BinaryOp.cpp +# binary/BinaryOp.cpp # ============================================================================= @@ -302,7 +302,7 @@ def register_comparison_ops(): # ============================================================================= -# BinaryOp.cpp (bitwise) +# binary/BinaryOp.cpp (bitwise) # ============================================================================= @@ -325,7 +325,7 @@ def register_bool_binary_ops(): # ============================================================================= -# BinaryScalarOp.cpp +# binary/BinaryScalarOp.cpp # ============================================================================= @@ -417,7 +417,7 @@ def register_softmax_cpp_ops(): # ============================================================================= -# MatMul.cpp +# gemm/matmul/MatMul.cpp # ============================================================================= @@ -437,7 +437,7 @@ def register_matmul_cpp_ops(): # ============================================================================= -# Linear.cpp +# gemm/linear/Linear.cpp # ============================================================================= @@ -457,7 +457,7 @@ def register_linear_cpp_ops(): # ============================================================================= -# QuantizedLinearQCSNW.cpp +# gemm/linear/qw/{q8csw,legacy}/*.cpp # ============================================================================= @@ -477,7 +477,7 @@ def register_quantizedlinearqcsnw_cpp_ops(): # ============================================================================= -# QuantizedLinear.cpp +# gemm/linear/qw/{q4gsw,dq8ca_q4gsw,legacy}/**/*.cpp # ============================================================================= @@ -627,7 +627,7 @@ def register_torchao_choose_qparams_affine(): # ============================================================================= -# Q8taBinary.cpp +# binary/q8ta/Q8taBinary.cpp # ============================================================================= @@ -896,7 +896,7 @@ def register_max_pool2d_with_indices(): # ============================================================================= -# Convolution.cpp +# convolution/Convolution.cpp # ============================================================================= @@ -989,7 +989,7 @@ def pick_conv_storage( # ============================================================================= -# Q8taConv2d*.cpp +# convolution/conv2d/q8ta/**/Q8taConv2d*.cpp # ============================================================================= @@ -1092,7 +1092,7 @@ def register_q8ta_conv2d_transposed_op(): # ============================================================================= -# Q8taLinear.cpp +# gemm/linear/q8ta/Q8taLinear.cpp # ============================================================================= diff --git a/backends/vulkan/runtime/gen_vulkan_spv.py b/backends/vulkan/runtime/gen_vulkan_spv.py index c9b054f1e73..5016e9c48db 100644 --- a/backends/vulkan/runtime/gen_vulkan_spv.py +++ b/backends/vulkan/runtime/gen_vulkan_spv.py @@ -748,6 +748,7 @@ def __init__( self.src_files: Dict[str, str] = {} self.template_yaml_files: List[str] = [] + self.yaml_relative_dirs: Dict[str, str] = {} self.addSrcAndYamlFiles(self.src_dir_paths) self.shader_template_params: Dict[Any, Any] = {} @@ -759,20 +760,30 @@ def __init__( def addSrcAndYamlFiles(self, src_dir_paths: List[str]) -> None: for src_path in src_dir_paths: + src_path = os.path.abspath(src_path) # Collect glsl source files src_files_list = glob.glob( os.path.join(src_path, "**", "*.[gh]lsl*"), recursive=True ) + glob.glob(os.path.join(src_path, "**", "*.h"), recursive=True) - for file in src_files_list: + for file in sorted(src_files_list): if len(file) > 1: - self.src_files[extract_filename(file, keep_ext=False)] = file + relative_path = os.path.relpath(file, src_path) + src_name = os.path.join( + os.path.dirname(relative_path), + extract_filename(relative_path, keep_ext=False), + ) + if src_name in self.src_files: + raise KeyError(f"{relative_path} source file is defined twice") + self.src_files[src_name] = file # Collect template yaml files yaml_files = glob.glob( os.path.join(src_path, "**", "*.yaml"), recursive=True ) - for file in yaml_files: + for file in sorted(yaml_files): if len(file) > 1: self.template_yaml_files.append(file) + relative_path = os.path.relpath(file, src_path) + self.yaml_relative_dirs[file] = os.path.dirname(relative_path) def generateVariantCombinations( # noqa: C901 self, @@ -839,14 +850,17 @@ def parseTemplateYaml(self, yaml_file: str) -> None: # noqa: C901 with open(yaml_file) as f: contents = yaml.load(f, Loader=UniqueKeyLoader) for template_name, params_dict in contents.items(): - if template_name in self.shader_template_params: - raise KeyError(f"{template_name} params file is defined twice") + template_path = os.path.normpath( + os.path.join(self.yaml_relative_dirs[yaml_file], template_name) + ) + if template_path in self.shader_template_params: + raise KeyError(f"{template_path} params file is defined twice") default_params = params_dict["parameter_names_with_default_values"] default_params["YAML_SRC_FULLPATH"] = yaml_file params_names = set(default_params.keys()).union({"NAME"}) - self.shader_template_params[template_name] = [] + self.shader_template_params[template_path] = [] default_iterated_params = params_dict.get( "generate_variant_forall", None @@ -907,7 +921,7 @@ def parseTemplateYaml(self, yaml_file: str) -> None: # noqa: C901 default_params_copy["NAME"] = variant_name default_params_copy["VARIANT_NAME"] = variant["NAME"] - self.shader_template_params[template_name].append( + self.shader_template_params[template_path].append( default_params_copy ) else: @@ -916,7 +930,7 @@ def parseTemplateYaml(self, yaml_file: str) -> None: # noqa: C901 if key not in reserved_yaml_keys: default_params_copy[key] = variant[key] - self.shader_template_params[template_name].append( + self.shader_template_params[template_path].append( default_params_copy ) @@ -944,18 +958,36 @@ def create_shader_params( return shader_params def constructOutputMap(self) -> None: + shader_names = set() + + def add_output( + output_name: str, + src_file_fullpath: str, + params: Dict[str, str], + ) -> None: + if output_name in self.output_file_map: + raise KeyError(f"{output_name} output file is defined twice") + if extract_extension(src_file_fullpath) == "glsl": + shader_name = os.path.basename(output_name) + if shader_name in shader_names: + raise KeyError(f"{shader_name} shader is defined twice") + shader_names.add(shader_name) + self.output_file_map[output_name] = (src_file_fullpath, params) + for src_name, params in self.shader_template_params.items(): for variant in params: src_file_fullpath = self.src_files[src_name] - - self.output_file_map[variant["NAME"]] = ( + output_name = os.path.join(os.path.dirname(src_name), variant["NAME"]) + add_output( + output_name, src_file_fullpath, self.create_shader_params(variant), ) for src_name, src_file_fullpath in self.src_files.items(): if src_name not in self.shader_template_params: - self.output_file_map[src_name] = ( + add_output( + src_name, src_file_fullpath, self.create_shader_params(), ) @@ -1054,21 +1086,55 @@ def file_has_changed(gen_file_path, cached_file_path): cached_checksum = self.get_md5_checksum(cached_file_path) return current_checksum != cached_checksum - def any_sources_changed(gen_file_path, output_dir): + def get_include_dirs(gen_file_path): + """ + Return the directories searched for a generated file's includes: its + own directory, then each ancestor up to output_dir, nearest first. + """ + include_dir = os.path.dirname(gen_file_path) + include_dirs = [include_dir] + while include_dir != output_dir: + parent_dir = os.path.dirname(include_dir) + if parent_dir == include_dir: + raise ValueError( + f"{gen_file_path} is outside output directory {output_dir}" + ) + include_dir = parent_dir + include_dirs.append(include_dir) + return include_dirs + + def any_sources_changed(gen_file_path, visited=None): """ Given the path to a generated source file, check the gen_file_meta dict to determine if the ANY of the source files contributing to the compilation of this file were changed since the last successful compilation. """ + if visited is None: + visited = set() + if gen_file_path in visited: + return False + if gen_file_path not in gen_file_meta: + return True + visited.add(gen_file_path) + gen_file_changed, includes_list = gen_file_meta[gen_file_path] - any_changed = gen_file_changed + if gen_file_changed: + return True + include_dirs = get_include_dirs(gen_file_path) for included_file in includes_list: - included_file_path = os.path.join(output_dir, included_file) - any_changed = any_changed or any_sources_changed( - included_file_path, output_dir + candidate_paths = [ + os.path.normpath(os.path.join(include_dir, included_file)) + for include_dir in include_dirs + ] + included_file_path = next( + (path for path in candidate_paths if path in gen_file_meta), + # Untracked includes must disable the cache. + candidate_paths[-1], ) + if any_sources_changed(included_file_path, visited): + return True - return any_changed + return False def generate_src_file(shader_paths_pair) -> Tuple[bool, List[str]]: """ @@ -1102,6 +1168,8 @@ def generate_src_file(shader_paths_pair) -> Tuple[bool, List[str]]: cached_gen_out_path = os.path.join( cache_dir, f"{src_file_name}.{out_file_ext}" ) + os.makedirs(os.path.dirname(gen_out_path), exist_ok=True) + os.makedirs(os.path.dirname(cached_gen_out_path), exist_ok=True) # Execute codegen to generate the output file with codecs.open(template_file_path, "r", encoding="utf-8") as input_file: @@ -1191,7 +1259,7 @@ def compile_spirv(shader_paths_pair) -> Tuple[str, str]: # in the cache. cached_spv_out_path = os.path.join(cache_dir, f"{src_file_name}.spv") - can_use_cached = not any_sources_changed(gen_out_path, output_dir) + can_use_cached = not any_sources_changed(gen_out_path) if can_use_cached and os.path.exists(cached_spv_out_path): shutil.copyfile(cached_spv_out_path, spv_out_path) return (spv_out_path, gen_out_path) @@ -1207,9 +1275,9 @@ def compile_spirv(shader_paths_pair) -> Tuple[str, str]: spv_out_path, "--target-env=vulkan{}".format(vk_version), "-Werror", - "-I", - output_dir, ] + for include_dir in get_include_dirs(gen_out_path): + cmd_base += ["-I", include_dir] cmd = cmd_base + self.glslc_flags try: diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_op_buffer.glsl b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_buffer.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_op_buffer.glsl rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_buffer.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_op_buffer.yaml b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_buffer.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_op_buffer.yaml rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_buffer.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_op_defs.glslh b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_defs.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_op_defs.glslh rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_defs.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_op_texture.glsl b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_texture.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_op_texture.glsl rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_texture.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_op_texture.yaml b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_texture.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_op_texture.yaml rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_op_texture.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_scalar_buffer.glsl b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_buffer.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_scalar_buffer.glsl rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_buffer.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_scalar_buffer.yaml b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_buffer.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_scalar_buffer.yaml rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_buffer.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_scalar_texture.glsl b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_texture.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_scalar_texture.glsl rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_texture.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/binary_scalar_texture.yaml b/backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_texture.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/binary_scalar_texture.yaml rename to backends/vulkan/runtime/graph/ops/glsl/binary/binary_scalar_texture.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_binary.glsl b/backends/vulkan/runtime/graph/ops/glsl/binary/q8ta/q8ta_binary.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_binary.glsl rename to backends/vulkan/runtime/graph/ops/glsl/binary/q8ta/q8ta_binary.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_binary.yaml b/backends/vulkan/runtime/graph/ops/glsl/binary/q8ta/q8ta_binary.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_binary.yaml rename to backends/vulkan/runtime/graph/ops/glsl/binary/q8ta/q8ta_binary.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_input_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_input_tile_load.glslh index 4456043bb9f..f1cafc503e3 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_input_tile_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_input_tile_load.glslh @@ -11,7 +11,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" VEC4_T load_fp_input_texel(const Conv2dTensorIndex tidx) { #ifdef INPUT_BUFFER diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv1d.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv1d.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv1d.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv1d.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv1d_dw.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_dw.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv1d_dw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_dw.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv1d_dw.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_dw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv1d_dw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_dw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv1d_pw.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_pw.glsl similarity index 96% rename from backends/vulkan/runtime/graph/ops/glsl/conv1d_pw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_pw.glsl index 4ebca126e03..41d422515f2 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv1d_pw.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_pw.glsl @@ -67,11 +67,11 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "linear_fp_input_tile.glslh" -#include "linear_fp_weight_tile.glslh" -#include "linear_fp_output_tile.glslh" -#include "linear_fp_packed_weight_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_weight_tile.glslh" +#include "gemm/tile_utils/linear_fp_output_tile.glslh" +#include "gemm/tile_utils/linear_fp_packed_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" // Conv1d pointwise is matrix multiplication with swapped texture coordinates. // Linear: input ivec3(k4, m, b), output ivec3(n4, m, b) [width-packed] diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv1d_pw.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_pw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv1d_pw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv1d/conv1d_pw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_common.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_common.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_common.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_common.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_output_tile.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_output_tile.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_output_tile.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_output_tile.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_output_tile.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_output_tile.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_output_tile.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_output_tile.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_sned_output_tile.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_sned_output_tile.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_sned_output_tile.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_sned_output_tile.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_sned_output_tile.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_sned_output_tile.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_sned_output_tile.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_dw_sned_output_tile.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_s1p0.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_s1p0.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_s1p0.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_s1p0.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_s1p0.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_s1p0.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_s1p0.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_s1p0.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_tiled.glsl similarity index 96% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_tiled.glsl index bd12d149a2c..e4e8ae7a5ff 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_tiled.glsl @@ -47,9 +47,9 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; ${layout_declare_spec_const(C, "int", "stride_1_padding_0", "0")} ${layout_declare_spec_const(C, "int", "activation_type", "0")} -#include "linear_fp_input_tile.glslh" -#include "linear_fp_packed_weight_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_packed_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" // Texel coordinates for the TILE_M spatial positions this invocation covers. // diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/conv2d_pw_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_gemm.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_gemm.glsl similarity index 98% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_gemm.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_gemm.glsl index 6eebdb6b25f..d704b2363bf 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_gemm.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_gemm.glsl @@ -114,9 +114,9 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; ${layout_declare_spec_const(C, "int", "activation_type", "0")} -#include "linear_fp_input_tile.glslh" -#include "linear_fp_packed_weight_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_packed_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" /* * Load TILE_M rows × TILE_K4 K-tiles of the im2col'd input. diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_gemm.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_gemm.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_gemm.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_gemm.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_im2col.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_im2col.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_im2col.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_im2col.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_im2col.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_im2col.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_im2col.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/conv2d_im2col.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_conv2d_gemm_weight.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/pack_weights/pack_conv2d_gemm_weight.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_conv2d_gemm_weight.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/pack_weights/pack_conv2d_gemm_weight.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_conv2d_gemm_weight.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/pack_weights/pack_conv2d_gemm_weight.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_conv2d_gemm_weight.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/im2col/pack_weights/pack_conv2d_gemm_weight.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_prepack_weights.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_dw_prepack_weights.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_prepack_weights.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_dw_prepack_weights.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_prepack_weights.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_dw_prepack_weights.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_prepack_weights.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_dw_prepack_weights.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_prepack_weights.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_prepack_weights.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_prepack_weights.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_prepack_weights.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_prepack_weights.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_prepack_weights.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_prepack_weights.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_prepack_weights.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_q8_utils.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/conv2d_dw_q8_utils.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_q8_utils.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/conv2d_dw_q8_utils.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_q8ta_q8csw_q8to.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/conv2d_dw_q8ta_q8csw_q8to.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_q8ta_q8csw_q8to.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/conv2d_dw_q8ta_q8csw_q8to.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_q8ta_q8csw_q8to.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/conv2d_dw_q8ta_q8csw_q8to.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_dw_q8ta_q8csw_q8to.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/conv2d_dw_q8ta_q8csw_q8to.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_im2col.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/im2col/q8ta_im2col.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_im2col.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/im2col/q8ta_im2col.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_im2col.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/im2col/q8ta_im2col.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_im2col.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/im2col/q8ta_im2col.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_dw_weights.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_dw_weights.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_dw_weights.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_dw_weights.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_dw_weights.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_dw_weights.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_dw_weights.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_dw_weights.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_weights.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_weights.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_weights.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_weights.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_weights.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_weights.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q8_conv2d_weights.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/pack_weights/pack_q8_conv2d_weights.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_dw.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_dw.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_dw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_dw.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_dw.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_dw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_dw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_dw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_pw.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_pw.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_pw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_pw.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_pw.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_pw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_pw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/q8ta_conv2d_pw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_transposed.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/transposed/q8ta_conv2d_transposed.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_transposed.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/transposed/q8ta_conv2d_transposed.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_transposed.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/transposed/q8ta_conv2d_transposed.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_conv2d_transposed.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/q8ta/transposed/q8ta_conv2d_transposed.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/col2im.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/col2im.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/col2im.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/col2im.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/col2im.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/col2im.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/col2im.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/col2im.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_load.glslh similarity index 98% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_load.glslh index b94bdaf506f..d156adc6e6a 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_load.glslh @@ -26,7 +26,7 @@ #include "indexing.glslh" #include "conv2d_common.glslh" #include "conv2d_fp_im2col_block.glslh" -#include "linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" VEC4_T load_input_texel(const TensorIndex4D tidx) { // Assumes batch size is 1 and channels packing diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_store.glslh similarity index 97% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block_store.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_store.glslh index 6c4dd7f0b52..8859b6c02e3 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_fp_im2col_block_store.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_fp_im2col_block_store.glslh @@ -23,7 +23,7 @@ #include "indexing.glslh" #include "conv2d_common.glslh" #include "conv2d_fp_im2col_block.glslh" -#include "linear_fp_output_tile.glslh" +#include "gemm/tile_utils/linear_fp_output_tile.glslh" // TODO: implement buffer support void write_output_texel(const VEC4_T out_texel, const TensorIndex4D tidx) { diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_input_block_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_input_block_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_input_block_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_input_block_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_input_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_input_tile_load.glslh similarity index 97% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_input_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_input_tile_load.glslh index 44aa09912ec..69407633e1f 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_input_tile_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_input_tile_load.glslh @@ -11,7 +11,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_int8_input_tile.glslh" +#include "gemm/tile_utils/linear_int8_input_tile.glslh" struct Int8InputTileIndex { #ifdef PACKED_INT8_INPUT_BUFFER diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_output_tile_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_output_tile_store.glslh similarity index 96% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_output_tile_store.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_output_tile_store.glslh index 27244f67953..e6c48d99462 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_int8_output_tile_store.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_int8_output_tile_store.glslh @@ -12,7 +12,7 @@ #extension GL_EXT_control_flow_attributes : require #include "conv2d_common.glslh" -#include "linear_int8_output_tile.glslh" +#include "gemm/tile_utils/linear_int8_output_tile.glslh" void store_packed_int8_output_tile( const Int8OutTile int8_tile, diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl similarity index 93% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl index f52241ee60f..bff8267c412 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_pw_q8ta_q8csw_q8to_tiled.glsl @@ -74,12 +74,12 @@ ${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")} #include "conv2d_int8_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_bias_load.glslh" -#include "linear_int8_output_tile_compute.glslh" +#include "gemm/tile_utils/linear_int8_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "gemm/tile_utils/linear_int_weight_sums_load.glslh" +#include "gemm/tile_utils/linear_fp_weight_scales_load.glslh" +#include "gemm/tile_utils/linear_fp_bias_load.glslh" +#include "gemm/tile_utils/linear_int8_output_tile_compute.glslh" #include "conv2d_int8_output_tile_store.glslh" void main() { diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_q8ta_q8csw_q8to_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_pw_q8ta_q8csw_q8to_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_pw_q8ta_q8csw_q8to_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_pw_q8ta_q8csw_q8to_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8_utils.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8_utils.glslh similarity index 98% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8_utils.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8_utils.glslh index 3fa03978fed..c30cf23aad8 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8_utils.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8_utils.glslh @@ -12,7 +12,7 @@ #extension GL_EXT_control_flow_attributes : require #extension GL_EXT_integer_dot_product : require -#include "linear_int_accumulator.glslh" +#include "gemm/tile_utils/linear_int_accumulator.glslh" struct Int8InputWindow1D { int[MAX_WINDOW_WIDTH] data; diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8csw_linear_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8csw_linear_tiled.glsl similarity index 92% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8csw_linear_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8csw_linear_tiled.glsl index a93b34a3e6e..f0dcaa88273 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8csw_linear_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8csw_linear_tiled.glsl @@ -63,12 +63,12 @@ ${layout_declare_spec_const(C, "int", "conv2d_params_logical_K_per_group", "1")} ${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")} -#include "linear_fp_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_bias_load.glslh" -#include "linear_fp_output_tile_fp_int8_compute.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_input_tile_load.glslh" +#include "gemm/tile_utils/linear_int8_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_weight_scales_load.glslh" +#include "gemm/tile_utils/linear_fp_bias_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_int8_compute.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" #include "conv2d_fp_im2col_block_store.glslh" void main() { diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8csw_linear_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8csw_linear_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8csw_linear_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8csw_linear_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_linear_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_linear_tiled.glsl similarity index 92% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_linear_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_linear_tiled.glsl index 3500f8392ad..dbbf3a08d2c 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_linear_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_linear_tiled.glsl @@ -71,12 +71,12 @@ ${layout_declare_spec_const(C, "int", "conv2d_params_logical_K_per_group", "1")} ${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")} -#include "linear_int8_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_fp_bias_load.glslh" +#include "gemm/tile_utils/linear_int8_input_tile_load.glslh" +#include "gemm/tile_utils/linear_int8_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_int_weight_sums_load.glslh" +#include "gemm/tile_utils/linear_fp_weight_scales_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "gemm/tile_utils/linear_fp_bias_load.glslh" #include "conv2d_fp_im2col_block_store.glslh" void main() { diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_linear_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_linear_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_linear_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_linear_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to.glsl similarity index 94% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to.glsl index 2c7f0aab981..4e3da29355b 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to.glsl @@ -77,12 +77,12 @@ ${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")} #include "im2col_packed_int8_utils.glslh" #include "conv2d_int8_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_bias_load.glslh" -#include "linear_int8_output_tile_compute.glslh" +#include "gemm/tile_utils/linear_int8_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "gemm/tile_utils/linear_int_weight_sums_load.glslh" +#include "gemm/tile_utils/linear_fp_weight_scales_load.glslh" +#include "gemm/tile_utils/linear_fp_bias_load.glslh" +#include "gemm/tile_utils/linear_int8_output_tile_compute.glslh" #include "conv2d_int8_output_tile_store.glslh" #include "conv2d_q8_utils.glslh" diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl similarity index 93% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl index f136c6ce99a..b46a0630862 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to_linear_tiled.glsl @@ -74,12 +74,12 @@ ${layout_declare_spec_const(C, "int", "conv2d_params_groups", "1")} #include "conv2d_int8_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_bias_load.glslh" -#include "linear_int8_output_tile_compute.glslh" +#include "gemm/tile_utils/linear_int8_weight_tile_load.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "gemm/tile_utils/linear_int_weight_sums_load.glslh" +#include "gemm/tile_utils/linear_fp_weight_scales_load.glslh" +#include "gemm/tile_utils/linear_fp_bias_load.glslh" +#include "gemm/tile_utils/linear_int8_output_tile_compute.glslh" #include "conv2d_int8_output_tile_store.glslh" void main() { diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to_linear_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to_linear_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv2d_q8ta_q8csw_q8to_linear_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/conv2d_q8ta_q8csw_q8to_linear_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/im2col.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/im2col.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/im2col.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/im2col.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/im2col_packed_int8.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col_packed_int8.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/im2col_packed_int8.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col_packed_int8.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/im2col_packed_int8.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col_packed_int8.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/im2col_packed_int8.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col_packed_int8.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/im2col_packed_int8_utils.glslh b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col_packed_int8_utils.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/im2col_packed_int8_utils.glslh rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/im2col_packed_int8_utils.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_im2col.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/quantize_and_pack_im2col.glsl similarity index 98% rename from backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_im2col.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/quantize_and_pack_im2col.glsl index a008866a779..1011b6d41ac 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_im2col.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/quantize_and_pack_im2col.glsl @@ -67,7 +67,7 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "conv2d_fp_im2col_block_load.glslh" -#include "linear_int8_input_block.glslh" +#include "gemm/tile_utils/linear_int8_input_block.glslh" void main() { // The quantized and packed im2col matrix can be conceptualized as a 2D matrix diff --git a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_im2col.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/quantize_and_pack_im2col.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_im2col.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/quantized_legacy/quantize_and_pack_im2col.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/conv_transpose2d.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/conv_transpose2d.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/conv_transpose2d.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/conv_transpose2d.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d_prepack_weights.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/pack_weights/conv_transpose2d_prepack_weights.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d_prepack_weights.glsl rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/pack_weights/conv_transpose2d_prepack_weights.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d_prepack_weights.yaml b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/pack_weights/conv_transpose2d_prepack_weights.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/conv_transpose2d_prepack_weights.yaml rename to backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/pack_weights/conv_transpose2d_prepack_weights.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/coopmat_mm.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/coopmat_mm.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/coopmat_mm.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/coopmat_mm.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/coopmat_mm.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/coopmat_mm.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/coopmat_mm.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/coopmat_mm.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_dW.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/backward/linear_dW.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_dW.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/backward/linear_dW.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_dW.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/backward/linear_dW.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_dW.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/backward/linear_dW.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_scalar.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_scalar.glsl similarity index 90% rename from backends/vulkan/runtime/graph/ops/glsl/linear_scalar.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_scalar.glsl index 9f5c2ee6a50..4ac5b9c88ee 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_scalar.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_scalar.glsl @@ -61,11 +61,11 @@ $else: layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "matmul_fp_mat1_tile_load.glslh" -#include "linear_fp_packed_weight_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "matmul_fp_out_tile_store.glslh" -#include "matmul_fp_bias_apply.glslh" +#include "tile_utils/matmul_fp_mat1_tile_load.glslh" +#include "tile_utils/linear_fp_packed_weight_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/matmul_fp_out_tile_store.glslh" +#include "tile_utils/matmul_fp_bias_apply.glslh" void main() { const int tile_idx_n = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_scalar.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_scalar.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_scalar.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_scalar.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_vec.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_vec.glsl similarity index 90% rename from backends/vulkan/runtime/graph/ops/glsl/linear_vec.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_vec.glsl index adf5272673e..f17520777f6 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_vec.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_vec.glsl @@ -62,11 +62,11 @@ $else: layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "matmul_fp_mat1_tile_load.glslh" -#include "linear_fp_packed_weight_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "matmul_fp_out_tile_store.glslh" -#include "matmul_fp_bias_apply.glslh" +#include "tile_utils/matmul_fp_mat1_tile_load.glslh" +#include "tile_utils/linear_fp_packed_weight_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/matmul_fp_out_tile_store.glslh" +#include "tile_utils/matmul_fp_bias_apply.glslh" void main() { const int tile_idx_n = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_vec.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_vec.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_vec.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/linear_vec.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_fp_linear_weight.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/pack_weights/pack_fp_linear_weight.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_fp_linear_weight.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/pack_weights/pack_fp_linear_weight.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_fp_linear_weight.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/pack_weights/pack_fp_linear_weight.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_fp_linear_weight.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/pack_weights/pack_fp_linear_weight.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_linear.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear.glsl similarity index 92% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_linear.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear.glsl index 98b0895d759..b09b593990c 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/q8ta_linear.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear.glsl @@ -57,12 +57,12 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "common.glslh" -#include "linear_int8_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_bias_load.glslh" +#include "tile_utils/linear_int8_input_tile_load.glslh" +#include "tile_utils/linear_int8_weight_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_int_weight_sums_load.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" void main() { const int out_tile_x = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_linear.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_linear.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_linear_gemv.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear_gemv.glsl similarity index 94% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_linear_gemv.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear_gemv.glsl index 9153cb0e294..7ae85518563 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/q8ta_linear_gemv.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear_gemv.glsl @@ -62,11 +62,11 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "dispatch.glslh" #include "common.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_bias_load.glslh" +#include "tile_utils/linear_int8_weight_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_int_weight_sums_load.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" // Array-of-arrays shared memory layout: partial_accums[lid][tile_n4]. // Each element is exactly 16 bytes (ivec4). This avoids the Samsung S25 diff --git a/backends/vulkan/runtime/graph/ops/glsl/q8ta_linear_gemv.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear_gemv.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q8ta_linear_gemv.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/q8ta/q8ta_linear_gemv.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_q4gsw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.glsl similarity index 85% rename from backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_q4gsw_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.glsl index a68a56c3713..5acab8e9a12 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_q4gsw_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.glsl @@ -60,19 +60,19 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; ${layout_declare_spec_const(C, "int", "apply_bias", "0")} ${layout_declare_spec_const(C, "int", "K4_per_group", "0")} -#include "linear_fp_input_tile_load.glslh" -#include "linear_int8_input_tile_load.glslh" -#include "linear_int8_input_scales_zps_load.glslh" -#include "linear_int4_weight_tile_load.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_int8_input_sums_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_fp_output_tile_int8_int4_compute.glslh" -#include "linear_fp_output_tile_fp_int4_compute.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "linear_fp_output_tile_store.glslh" -#include "linear_fp_bias_load.glslh" +#include "tile_utils/linear_fp_input_tile_load.glslh" +#include "tile_utils/linear_int8_input_tile_load.glslh" +#include "tile_utils/linear_int8_input_scales_zps_load.glslh" +#include "tile_utils/linear_int4_weight_tile_load.glslh" +#include "tile_utils/linear_int_weight_sums_load.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_int8_input_sums_load.glslh" +#include "tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "tile_utils/linear_fp_output_tile_int8_int4_compute.glslh" +#include "tile_utils/linear_fp_output_tile_fp_int4_compute.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/linear_fp_output_tile_store.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" void main() { const int out_tile_x = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_q4gsw_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_q4gsw_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_qw_coopmat.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_qw_coopmat.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_qw_coopmat.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_qw_coopmat.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_qw_coopmat.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_qw_coopmat.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_qw_coopmat.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_qw_coopmat.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q8ta_q8csw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_q8ta_q8csw_tiled.glsl similarity index 89% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q8ta_q8csw_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_q8ta_q8csw_tiled.glsl index e3f49fac475..20003489dca 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_q8ta_q8csw_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_q8ta_q8csw_tiled.glsl @@ -50,13 +50,13 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "linear_int8_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_output_tile_int8_int8_compute.glslh" -#include "linear_fp_output_tile_store.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_int_weight_sums_load.glslh" -#include "linear_fp_bias_load.glslh" +#include "tile_utils/linear_int8_input_tile_load.glslh" +#include "tile_utils/linear_int8_weight_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_int8_int8_compute.glslh" +#include "tile_utils/linear_fp_output_tile_store.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_int_weight_sums_load.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" void main() { // Each thread writes out a 4 wide x 4 high tile of output values diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q8ta_q8csw_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_q8ta_q8csw_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q8ta_q8csw_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_q8ta_q8csw_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_coop.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_coop.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_coop.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_coop.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_coop.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_coop.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_coop.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_coop.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_tiled.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_tiled.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qcsnw_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/legacy/linear_qcsnw_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_linear_weight.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/pack_weights/pack_q8_linear_weight.glsl similarity index 97% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q8_linear_weight.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/pack_weights/pack_q8_linear_weight.glsl index 7b17dbe4990..765c8a164e1 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_linear_weight.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/pack_weights/pack_q8_linear_weight.glsl @@ -26,7 +26,7 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "common.glslh" -#include "linear_int8_weight_block.glslh" +#include "tile_utils/linear_int8_weight_block.glslh" void main() { // The size of the source weight tensor is [W=K, H=N]. Each shader invocation diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q8_linear_weight.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/pack_weights/pack_q8_linear_weight.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q8_linear_weight.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/pack_weights/pack_q8_linear_weight.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_backward.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_backward.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_backward.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_backward.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_backward.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_backward.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_backward.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_backward.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_requant.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_requant.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_requant.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_requant.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_requant.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_requant.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_requant.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/backward/q4gsw_requant.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_coop.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_coop.glsl similarity index 92% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_coop.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_coop.glsl index 0f88dc59702..8a2c9f8bdcf 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_coop.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_coop.glsl @@ -63,13 +63,13 @@ ${layout_declare_spec_const(C, "int", "apply_bias", "0")} ${layout_declare_spec_const(C, "int", "K4_per_group", "0")} #include "common.glslh" -#include "linear_fp_input_tile_load.glslh" -#include "linear_int4_weight_tile_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_output_tile_fp_int4_compute.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "linear_fp_output_tile_store.glslh" -#include "linear_fp_bias_load.glslh" +#include "tile_utils/linear_fp_input_tile_load.glslh" +#include "tile_utils/linear_int4_weight_tile_load.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_fp_output_tile_fp_int4_compute.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/linear_fp_output_tile_store.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" shared FPOutTile partial_sums[WGS]; diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_coop.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_coop.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_coop.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_coop.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_tiled.glsl similarity index 89% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_tiled.glsl index f79512ac7ee..9bfa875d58d 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_tiled.glsl @@ -49,13 +49,13 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; ${layout_declare_spec_const(C, "int", "apply_bias", "0")} ${layout_declare_spec_const(C, "int", "K4_per_group", "0")} -#include "linear_fp_input_tile_load.glslh" -#include "linear_int4_weight_tile_load.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_bias_load.glslh" -#include "linear_fp_output_tile_fp_int4_compute.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "linear_fp_output_tile_store.glslh" +#include "tile_utils/linear_fp_input_tile_load.glslh" +#include "tile_utils/linear_int4_weight_tile_load.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" +#include "tile_utils/linear_fp_output_tile_fp_int4_compute.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/linear_fp_output_tile_store.glslh" void main() { const int out_tile_x = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q4gsw_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_q4gsw_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qw_coopmat.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_qw_coopmat.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qw_coopmat.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_qw_coopmat.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_qw_coopmat.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_qw_coopmat.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_qw_coopmat.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/linear_qw_coopmat.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight.glsl similarity index 97% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight.glsl index b9f5c994910..2b2cf023b5c 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight.glsl @@ -25,7 +25,7 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "common.glslh" -#include "linear_int4_weight_block.glslh" +#include "tile_utils/linear_int4_weight_block.glslh" void main() { const int k8 = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight__w_4x8.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight__w_4x8.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight__w_4x8.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight__w_4x8.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight__w_4x8.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight__w_4x8.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_q4_linear_weight__w_4x8.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/pack_weights/pack_q4_linear_weight__w_4x8.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__tin__w_4x8.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__tin__w_4x8.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__tin__w_4x8.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__tin__w_4x8.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__tin__w_4x8.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__tin__w_4x8.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__tin__w_4x8.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__tin__w_4x8.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__w_4x8.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__w_4x8.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__w_4x8.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__w_4x8.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__w_4x8.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__w_4x8.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemm__w_4x8.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemm__w_4x8.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemv_coop__w_4x8.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemv_coop__w_4x8.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemv_coop__w_4x8.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemv_coop__w_4x8.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemv_coop__w_4x8.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemv_coop__w_4x8.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/q4gsw_linear_gemv_coop__w_4x8.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q4gsw/q4gsw_linear_gemv_coop__w_4x8.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q8csw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q8csw/linear_q8csw_tiled.glsl similarity index 87% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q8csw_tiled.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q8csw/linear_q8csw_tiled.glsl index c8dc6bfeec4..51a73d1126f 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/linear_q8csw_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q8csw/linear_q8csw_tiled.glsl @@ -44,14 +44,14 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; ${layout_declare_spec_const(C, "uint", "apply_bias", "0")} -#include "linear_fp_input_tile_load.glslh" -#include "linear_int8_weight_tile_load.glslh" -#include "linear_fp_weight_tile.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "linear_fp_output_tile_fp_int8_compute.glslh" -#include "linear_fp_output_tile_store.glslh" -#include "linear_fp_weight_scales_load.glslh" -#include "linear_fp_bias_load.glslh" +#include "tile_utils/linear_fp_input_tile_load.glslh" +#include "tile_utils/linear_int8_weight_tile_load.glslh" +#include "tile_utils/linear_fp_weight_tile.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/linear_fp_output_tile_fp_int8_compute.glslh" +#include "tile_utils/linear_fp_output_tile_store.glslh" +#include "tile_utils/linear_fp_weight_scales_load.glslh" +#include "tile_utils/linear_fp_bias_load.glslh" void main() { // Each thread writes out a 4 wide x 4 high tile of output values diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_q8csw_tiled.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q8csw/linear_q8csw_tiled.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_q8csw_tiled.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/q8csw/linear_q8csw_tiled.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_scalar.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_scalar.glsl similarity index 90% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_scalar.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_scalar.glsl index a3f68dda839..bfffe5fa38d 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/matmul_scalar.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_scalar.glsl @@ -54,11 +54,11 @@ $if HAS_BIAS: layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "matmul_fp_mat1_tile_load.glslh" -#include "matmul_fp_mat2_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "matmul_fp_out_tile_store.glslh" -#include "matmul_fp_bias_apply.glslh" +#include "tile_utils/matmul_fp_mat1_tile_load.glslh" +#include "tile_utils/matmul_fp_mat2_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/matmul_fp_out_tile_store.glslh" +#include "tile_utils/matmul_fp_bias_apply.glslh" void main() { const int tile_idx_n = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_scalar.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_scalar.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_scalar.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_scalar.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_vec.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_vec.glsl similarity index 90% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_vec.glsl rename to backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_vec.glsl index 9d4f6f04aba..c14861170c2 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/matmul_vec.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_vec.glsl @@ -55,11 +55,11 @@ $if HAS_BIAS: layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "matmul_fp_mat1_tile_load.glslh" -#include "matmul_fp_mat2_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" -#include "matmul_fp_out_tile_store.glslh" -#include "matmul_fp_bias_apply.glslh" +#include "tile_utils/matmul_fp_mat1_tile_load.glslh" +#include "tile_utils/matmul_fp_mat2_tile_load.glslh" +#include "tile_utils/linear_fp_output_tile_fp_compute.glslh" +#include "tile_utils/matmul_fp_out_tile_store.glslh" +#include "tile_utils/matmul_fp_bias_apply.glslh" void main() { const int tile_idx_n = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_vec.yaml b/backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_vec.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_vec.yaml rename to backends/vulkan/runtime/graph/ops/glsl/gemm/matmul/matmul_vec.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_common.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_common.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_common.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_common.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_bias_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_bias_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_bias_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_bias_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_input_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_input_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_input_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_input_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_input_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_input_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_input_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_input_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_fp_compute.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_fp_compute.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_fp_int4_compute.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_fp_int4_compute.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_fp_int4_compute.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_fp_int4_compute.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_fp_int8_compute.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_fp_int8_compute.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_fp_int8_compute.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_fp_int8_compute.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_int8_int4_compute.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_int8_int4_compute.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_int8_int4_compute.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_int8_int4_compute.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_int8_int8_compute.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_int8_int8_compute.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_int8_int8_compute.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_int8_int8_compute.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_store.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_output_tile_store.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_output_tile_store.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_packed_weight_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_packed_weight_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_packed_weight_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_packed_weight_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_per_out_channel_params.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_per_out_channel_params.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_per_out_channel_params.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_per_out_channel_params.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_weight_scales_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_weight_scales_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_weight_scales_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_weight_scales_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_fp_weight_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_weight_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_fp_weight_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_fp_weight_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int4_weight_block.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int4_weight_block.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int4_weight_block.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int4_weight_block.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int4_weight_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int4_weight_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int4_weight_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int4_weight_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int4_weight_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int4_weight_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int4_weight_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int4_weight_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_block.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_block.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_block.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_block.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_scales_zps.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_scales_zps.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_scales_zps.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_scales_zps.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_scales_zps_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_scales_zps_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_scales_zps_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_scales_zps_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_sums_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_sums_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_sums_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_sums_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_sums_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_sums_store.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_sums_store.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_sums_store.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_input_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_input_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_output_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_output_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_output_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_output_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_output_tile_compute.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_output_tile_compute.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_output_tile_compute.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_output_tile_compute.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_weight_block.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_weight_block.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_weight_block.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_weight_block.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_weight_tile.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_weight_tile.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_weight_tile.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_weight_tile.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int8_weight_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_weight_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int8_weight_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int8_weight_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int_accumulator.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_accumulator.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int_accumulator.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_accumulator.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int_per_in_channel_params.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_per_in_channel_params.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int_per_in_channel_params.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_per_in_channel_params.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int_per_out_channel_params.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_per_out_channel_params.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int_per_out_channel_params.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_per_out_channel_params.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/linear_int_weight_sums_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_weight_sums_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/linear_int_weight_sums_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/linear_int_weight_sums_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_fp_bias_apply.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_bias_apply.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_fp_bias_apply.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_bias_apply.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_fp_mat1_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_mat1_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_fp_mat1_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_mat1_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_fp_mat2_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_mat2_tile_load.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_fp_mat2_tile_load.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_mat2_tile_load.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/matmul_fp_out_tile_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_out_tile_store.glslh similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/matmul_fp_out_tile_store.glslh rename to backends/vulkan/runtime/graph/ops/glsl/gemm/tile_utils/matmul_fp_out_tile_store.glslh diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_int4_linear_weight_transposed_interleaved.glsl b/backends/vulkan/runtime/graph/ops/glsl/pack_weights/pack_int4_linear_weight_transposed_interleaved.glsl similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_int4_linear_weight_transposed_interleaved.glsl rename to backends/vulkan/runtime/graph/ops/glsl/pack_weights/pack_int4_linear_weight_transposed_interleaved.glsl diff --git a/backends/vulkan/runtime/graph/ops/glsl/pack_int4_linear_weight_transposed_interleaved.yaml b/backends/vulkan/runtime/graph/ops/glsl/pack_weights/pack_int4_linear_weight_transposed_interleaved.yaml similarity index 100% rename from backends/vulkan/runtime/graph/ops/glsl/pack_int4_linear_weight_transposed_interleaved.yaml rename to backends/vulkan/runtime/graph/ops/glsl/pack_weights/pack_int4_linear_weight_transposed_interleaved.yaml diff --git a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w.glsl b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w.glsl index 9d65d7d534f..41414a22dfd 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w.glsl @@ -38,8 +38,8 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "linear_int8_input_block.glslh" -#include "linear_fp_input_tile_load.glslh" +#include "gemm/tile_utils/linear_int8_input_block.glslh" +#include "gemm/tile_utils/linear_fp_input_tile_load.glslh" void main() { // Each input block contains 4x4 int8 quantized values, which are packed into diff --git a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w_with_group_sums.glsl b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w_with_group_sums.glsl index d5f14f70b62..407c49fddf4 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w_with_group_sums.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4h4w_with_group_sums.glsl @@ -48,9 +48,9 @@ shared ivec4 shared_sums[NUM_GROUPS_PER_WG][NUM_WORKERS_PER_GROUP]; #define TILE_M 4 -#include "linear_int8_input_block.glslh" -#include "linear_int8_input_scales_zps_load.glslh" -#include "linear_fp_input_tile_load.glslh" +#include "gemm/tile_utils/linear_int8_input_block.glslh" +#include "gemm/tile_utils/linear_int8_input_scales_zps_load.glslh" +#include "gemm/tile_utils/linear_fp_input_tile_load.glslh" void main() { const int group_idx = int(gl_GlobalInvocationID.x); diff --git a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4w4c.glsl b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4w4c.glsl index 205929d7456..3b5be301425 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4w4c.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/quantize_and_pack_4w4c.glsl @@ -28,7 +28,7 @@ $if INPUT_STORAGE == "buffer": layout(std430) buffer; -#include "conv2d_common.glslh" +#include "convolution/conv2d/conv2d_common.glslh" ${layout_declare_tensor(B, "w", "t_packed_int8_input", "int", OUTPUT_STORAGE, is_scalar_array=False)} ${layout_declare_tensor(B, "r", "t_fp_input", DTYPE, INPUT_STORAGE)} @@ -43,7 +43,7 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "conv2d_fp_input_tile_load.glslh" -#include "linear_int8_input_block.glslh" +#include "gemm/tile_utils/linear_int8_input_block.glslh" void store_packed_int8_block( const Conv2dBlockIndex block_idx, diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_coop.glsl b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_coop.glsl index b7f14f435fa..ea1a80d6388 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_coop.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_coop.glsl @@ -51,7 +51,7 @@ ${layout_declare_spec_const(C, "float", "inv_scale", "1.0")} #include "sdpa_fp_q_projected_tile_load.glslh" #include "sdpa_fp_k_cache_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" #include "sdpa_fp_attn_weight_tile_store.glslh" shared FPOutTile partial_sums[NUM_WORKERS_PER_OUT]; diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_tiled.glsl index 662d5edde68..a0feba5bb15 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_attn_weights_tiled.glsl @@ -70,7 +70,7 @@ ${layout_declare_spec_const(C, "float", "inv_scale", "1.0")} #include "sdpa_fp_q_projected_tile_load.glslh" #include "sdpa_fp_k_cache_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" #include "sdpa_fp_attn_weight_tile_store.glslh" /* diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_coop.glsl b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_coop.glsl index 7f873ce0718..d1aab16290f 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_coop.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_coop.glsl @@ -64,7 +64,7 @@ $if GQA: #include "sdpa_fp_attn_weight_tile_load.glslh" #include "sdpa_fp_v_cache_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" #include "sdpa_fp_out_tile_store.glslh" #ifdef GQA diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_tiled.glsl index 9f8f2dbc231..d191401296c 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_compute_out_tiled.glsl @@ -56,7 +56,7 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; #include "sdpa_fp_attn_weight_tile_load.glslh" #include "sdpa_fp_v_cache_tile_load.glslh" -#include "linear_fp_output_tile_fp_compute.glslh" +#include "gemm/tile_utils/linear_fp_output_tile_fp_compute.glslh" #include "sdpa_fp_out_tile_store.glslh" /* diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_load.glslh index 829f03beb60..af639e77371 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_load.glslh @@ -27,7 +27,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" LINEAR_FP_INPUT_TILE_VEC4_T load_attn_weight_c4( const int c4, diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_store.glslh index 83d81ddf812..78a7df1d4be 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_store.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_attn_weight_tile_store.glslh @@ -32,7 +32,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_output_tile.glslh" +#include "gemm/tile_utils/linear_fp_output_tile.glslh" // ============================================================ // Shared store helpers (buffer or texture; fp32 or input-dtype) diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_k_cache_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_k_cache_tile_load.glslh index 65d08755528..555561581c7 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_k_cache_tile_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_k_cache_tile_load.glslh @@ -31,7 +31,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_weight_tile.glslh" +#include "gemm/tile_utils/linear_fp_weight_tile.glslh" // Determine whether buffer mode is active. Both K_CACHE_BUFFER (LLM) and // K_BUFFER (fused) activate the buffer path. diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_out_tile_store.glslh b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_out_tile_store.glslh index 382747bf9a4..2c04170bab4 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_out_tile_store.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_out_tile_store.glslh @@ -30,7 +30,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_output_tile.glslh" +#include "gemm/tile_utils/linear_fp_output_tile.glslh" void store_out_d4( const LINEAR_FP_OUTPUT_TILE_VEC4_T out_texel, diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_q_projected_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_q_projected_tile_load.glslh index 752762b623d..40ed5b9e181 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_q_projected_tile_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_q_projected_tile_load.glslh @@ -28,7 +28,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" LINEAR_FP_INPUT_TILE_VEC4_T load_q_projected_d4( const int d4, diff --git a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_v_cache_tile_load.glslh b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_v_cache_tile_load.glslh index 98516744b44..fc2b280dedd 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_v_cache_tile_load.glslh +++ b/backends/vulkan/runtime/graph/ops/glsl/sdpa_fp_v_cache_tile_load.glslh @@ -30,7 +30,7 @@ #extension GL_EXT_control_flow_attributes : require -#include "linear_fp_weight_tile.glslh" +#include "gemm/tile_utils/linear_fp_weight_tile.glslh" #ifndef SDPA_V_BUF #define SDPA_V_BUF t_v diff --git a/backends/vulkan/runtime/graph/ops/glsl/unpack_4w4c_and_dequantize.glsl b/backends/vulkan/runtime/graph/ops/glsl/unpack_4w4c_and_dequantize.glsl index 3baa21f54bf..41a106ab5ff 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/unpack_4w4c_and_dequantize.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/unpack_4w4c_and_dequantize.glsl @@ -28,7 +28,7 @@ $if INPUT_STORAGE == "buffer": layout(std430) buffer; -#include "conv2d_common.glslh" +#include "convolution/conv2d/conv2d_common.glslh" ${layout_declare_tensor(B, "w", "t_fp_output", DTYPE, OUTPUT_STORAGE)} ${layout_declare_tensor(B, "r", "t_packed_int8_output", "int", INPUT_STORAGE, is_scalar_array=False)} @@ -42,8 +42,8 @@ layout(push_constant) uniform restrict Block { layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; -#include "linear_fp_input_tile.glslh" -#include "linear_int8_input_tile.glslh" +#include "gemm/tile_utils/linear_fp_input_tile.glslh" +#include "gemm/tile_utils/linear_int8_input_tile.glslh" void load_packed_int8_tile( out Int8InputTile int8_tile, diff --git a/backends/vulkan/runtime/graph/ops/impl/Convolution.cpp b/backends/vulkan/runtime/graph/ops/impl/Convolution.cpp deleted file mode 100644 index 272e886b7b1..00000000000 --- a/backends/vulkan/runtime/graph/ops/impl/Convolution.cpp +++ /dev/null @@ -1,871 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include - -#include - -#include -#include -#include - -#include - -#include -#include - -#include - -namespace vkcompute { - -enum class Conv2dMethod : uint8_t { - Depthwise, - Pointwise, - SlidingWindow, - Transposed, -}; - -void resize_conv2d_node( - ComputeGraph* graph, - const std::vector& args, - const std::vector& extra_args) { - const ValueRef out = args.at(0).refs.at(0); - const ValueRef self = args.at(1).refs.at(0); - - size_t ndim = graph->dim_of(self); - std::vector new_out_sizes(ndim); - const bool transposed = graph->get_bool(extra_args.at(4)); - - std::vector self_sizes = graph->sizes_of(self); - // Batch, Channel - if (ndim == 4) { - new_out_sizes.at(ndim - 4) = self_sizes.at(ndim - 4); - } - - TensorRefPtr weight_ref = graph->get_tref(extra_args.at(0)); - const auto& weight_sizes = weight_ref->sizes; - new_out_sizes.at(ndim - 3) = - transposed ? weight_sizes.at(ndim - 3) : weight_sizes.at(ndim - 4); - - // Height, Width - const auto& new_out_sizes_hw = calc_out_sizes_hw( - *graph, - self_sizes, - extra_args.at(0), - /*kernel_size_only = */ false, - {extra_args.at(1), extra_args.at(2), extra_args.at(3), extra_args.at(5)}, - transposed); - new_out_sizes.at(ndim - 2) = new_out_sizes_hw.at(0); - new_out_sizes.at(ndim - 1) = new_out_sizes_hw.at(1); - - graph->virtual_resize(out, new_out_sizes); -} - -void resize_conv1d_node( - ComputeGraph* graph, - const std::vector& args, - const std::vector& extra_args) { - const ValueRef out = args.at(0).refs.at(0); - const ValueRef self = args.at(1).refs.at(0); - TensorRefPtr weight_ref = graph->get_tref(extra_args.at(0)); - - const int64_t stride_size = graph->get_int_list(extra_args.at(1))->at(0); - const int64_t padding_size = graph->get_int_list(extra_args.at(2))->at(0); - const int64_t dilation_size = graph->get_int_list(extra_args.at(3))->at(0); - - const std::vector& weight_sizes = weight_ref->sizes; - - const std::vector in_sizes = graph->sizes_of(self); - const size_t ndim = in_sizes.size(); - std::vector new_out_sizes(ndim); - - const int64_t kernel_size = weight_sizes.at(2); - const int64_t in_length = in_sizes.at(2); - - new_out_sizes.at(0) = in_sizes.at(0); - new_out_sizes.at(1) = weight_sizes.at(0); - new_out_sizes.at(2) = calc_out_size( - in_length, kernel_size, stride_size, padding_size, dilation_size, false); - - graph->virtual_resize(out, new_out_sizes); -} - -ValueRef prepack_biases( - ComputeGraph& graph, - const ValueRef vref, - const ValueRef weight, - const bool transposed, - const utils::StorageType storage_type, - const utils::GPUMemoryLayout memory_layout) { - auto sizes = graph.sizes_of(weight); - const int64_t out_channels = transposed ? sizes.at(1) : sizes.at(0); - - ValueRef v = graph.add_tensor( - {out_channels}, graph.dtype_of(weight), storage_type, memory_layout); - - vkapi::ShaderInfo shader = - get_nchw_to_tensor_shader(graph, v, graph.get_staging_dtype_for(weight)); - - vkapi::ParamsBindList param_buffers = {}; - if (graph.is_buffer_storage(v)) { - param_buffers.append(graph.buffer_meta_ubo(v)); - } else { - param_buffers.append(graph.texture_meta_ubo(v)); - } - - std::vector pcs; - if (graph.is_buffer_storage(v)) { - pcs = {graph.sizes_pc_of(v), graph.strides_pc_of(v), graph.numel_pc_of(v)}; - } - - graph.prepack_nodes().emplace_back(new PrepackNode( - graph, - shader, - graph.create_gwg(v), - graph.create_lwg(v), - vref, - v, - param_buffers, - // Specialization constants - {graph.hashed_layout_of(v)}, - pcs)); - - return v; -} - -vkapi::ShaderInfo get_conv2d_shader( - ComputeGraph& graph, - const ValueRef out, - const bool prepack_weights, - const Conv2dMethod method, - const ValueRef weight, - const bool clamp_out = false, - const bool stride_equals_dilation = false, - const bool stride_1_padding_0 = false) { - std::string kernel_name; - kernel_name.reserve(kShaderNameReserve); - switch (method) { - case Conv2dMethod::Depthwise: - kernel_name = "conv2d_dw"; - break; - case Conv2dMethod::Pointwise: - if (prepack_weights) { - kernel_name = "conv2d"; - } else { - kernel_name = stride_1_padding_0 ? "conv2d_pw_s1p0" : "conv2d_pw"; - } - break; - case Conv2dMethod::SlidingWindow: - kernel_name = "conv2d"; - break; - case Conv2dMethod::Transposed: - kernel_name = "conv_transpose2d"; - break; - } - if (prepack_weights) { - kernel_name += "_prepack_weights"; - } else if (clamp_out) { - kernel_name += "_clamp"; - } - add_dtype_suffix(kernel_name, graph.dtype_of(out)); - - if (prepack_weights) { - add_dtype_suffix(kernel_name, graph.get_staging_dtype_for(weight)); - } - - return VK_KERNEL_FROM_STR(kernel_name); -} - -std::vector get_final_sizes( - const std::vector& original_sizes, - const Conv2dMethod method) { - int64_t batch_padded = utils::align_up_4(utils::val_at(-4, original_sizes)); - int64_t channels_padded = - utils::align_up_4(utils::val_at(-3, original_sizes)); - int64_t height = utils::val_at(-2, original_sizes); - int64_t width = utils::val_at(-1, original_sizes); - - switch (method) { - case Conv2dMethod::Depthwise: - return std::vector{4, batch_padded / 4, height * width}; - case Conv2dMethod::Pointwise: - case Conv2dMethod::SlidingWindow: - return std::vector{ - 4, batch_padded * height / 4, channels_padded * width}; - case Conv2dMethod::Transposed: - return std::vector{ - 4, channels_padded * height / 4, batch_padded * width}; - } -} - -ValueRef prepack_weights( - ComputeGraph& graph, - const ValueRef vref, - const Conv2dMethod method) { - const auto original_sizes = graph.sizes_of(vref); - const auto final_sizes = get_final_sizes(original_sizes, method); - - ValueRef v = graph.add_tensor( - final_sizes, - graph.dtype_of(vref), - utils::kTexture2D, - utils::kChannelsPacked); - - vkapi::ShaderInfo shader = - get_conv2d_shader(graph, v, /*prepack_weights = */ true, method, vref); - - const auto original_sizes_pc = - utils::make_ivec4(original_sizes, /*reverse = */ true); - graph.prepack_nodes().emplace_back(new PrepackNode( - graph, - shader, - graph.create_gwg(v), - graph.create_lwg(v), - vref, - v, - {}, - // Specialization constants - {graph.packed_dim_of(v)}, - {graph.sizes_pc_of(v), - PushConstantDataInfo(&original_sizes_pc, sizeof(original_sizes_pc))})); - - return v; -} - -void check_conv_args( - ComputeGraph& graph, - const ValueRef in, - const ValueRef out) { - VK_CHECK_COND(graph.packed_dim_of(in) == WHCN::kChannelsDim); - VK_CHECK_COND(graph.packed_dim_of(out) == WHCN::kChannelsDim); -} - -struct Conv2dParams final { - utils::ivec2 overlay_region; - int in_group_size; -}; - -struct OutputParams final { - float out_min; - float out_max; -}; - -Conv2dParams create_conv2d_params( - ComputeGraph& graph, - const ValueRef weight, - const Kernel2dParams& p, - const bool transposed) { - const auto& overlay_region = utils::make_ivec2({ - p.kernel_size[0] + (p.kernel_size[0] - 1) * (p.dilation[0] - 1), - p.kernel_size[1] + (p.kernel_size[1] - 1) * (p.dilation[1] - 1), - }); - const auto weight_sizes = graph.sizes_of(weight); - const int32_t in_group_size = utils::safe_downcast( - utils::align_up_4(transposed ? weight_sizes.at(0) : weight_sizes.at(1))); - return {overlay_region, in_group_size}; -} - -void check_conv2d_params(const Kernel2dParams& p, const bool transposed) { - if (transposed) { - if (p.dilation[0] > 1 || p.dilation[1] > 1) { - VK_THROW( - "aten.convolution.default: transposed = true, dilation > 1 is not supported yet!"); - } - } -} - -Conv2dMethod get_conv2d_method( - ComputeGraph& graph, - const ValueRef weight, - const int64_t groups, - const bool transposed) { - const auto weight_sizes = graph.sizes_of(weight); - if (!transposed && weight_sizes.at(0) == groups && weight_sizes.at(1) == 1) { - return Conv2dMethod::Depthwise; - } - if (transposed) { - return Conv2dMethod::Transposed; - } - if (weight_sizes.at(2) == 1 && weight_sizes.at(3) == 1) { - return Conv2dMethod::Pointwise; - } - return Conv2dMethod::SlidingWindow; -} - -// Decide whether a SlidingWindow conv2d should be computed via the -// im2col + GEMM path (conv2d_gemm_impl) instead of the direct convolution -// shader. Across 26 configs on Mali-G715 (buffer path) and Adreno SM8650 -// (texture path): FP32 cases were numerically verified against the reference; -// FP16 cases were routing/dispatch-validated only (the reference is float-only -// for the large shapes, so FP16 outputs were not numerically checked). -// -// Only called for SlidingWindow conv2d (1x1 is routed to conv2d_pw and -// Depthwise/Transposed are handled before the call site). -// -// Preconditions (fall back to direct conv if any fail — the im2col path is -// either not applicable or not beneficial): -// - groups == 1 -// - dilation == 1 (all dims) -// -// Selection rule: use im2col once the output channel count is large enough to -// amortize the fixed ~N*K_total im2col gather cost. -constexpr int64_t kIm2colMinCOut = 128; - -// A cheap gather is a second, independent reason to take im2col. The gather -// materializes an N x K_total matrix before the GEMM runs, and a large gather -// is fine when c_out is large because the GEMM reads it back c_out times. When -// c_out is small it is only worth paying if the matrix is small outright, which -// is where the direct shader on Mali loses badly: an 80x80 3x3 conv wants ~4M -// elements, a 640x640 9x9 conv wants ~100M. -constexpr int64_t kIm2colMaxCheapGatherElements = 32 * 1024 * 1024; - -bool should_use_conv2d_im2col( - ComputeGraph& graph, - const ValueRef weight_data, - const int64_t groups_val, - const Kernel2dParams& kernel_params, - const ValueRef out) { - if (groups_val != 1) { - return false; - } - if (kernel_params.dilation[0] != 1 || kernel_params.dilation[1] != 1) { - return false; - } - const auto weight_sizes = graph.sizes_of(weight_data); - const int64_t c_out = weight_sizes.at(0); - const int64_t k_total = - weight_sizes.at(1) * weight_sizes.at(2) * weight_sizes.at(3); - - const auto out_sizes = graph.sizes_of(out); - const size_t ndim = out_sizes.size(); - const int64_t n = out_sizes.at(ndim - 1) * out_sizes.at(ndim - 2); - return c_out >= kIm2colMinCOut || - (graph.device_is_mali() && n * k_total <= kIm2colMaxCheapGatherElements); -} - -GlobalWorkGrid create_conv2d_gwg( - ComputeGraph& graph, - const Conv2dMethod method, - const ValueRef out, - const ValueRef weight_data, - const bool stride_equals_dilation) { - if (method == Conv2dMethod::Pointwise) { - const utils::uvec3 image_extents = graph.logical_limits_of(out); - return GlobalWorkGrid( - {utils::div_up(image_extents[0u], 1u), - utils::div_up(image_extents[1u], 4u), - image_extents[2u]}, - kTiledWorkGrid); - } else { - return graph.create_gwg(out); - } -} - -// Determines which convolution method a dispatch uses. -// -// Depthwise and transposed convolutions have shader names of their own, but -// the name alone cannot separate pointwise from sliding window: the sliding -// window shader is itself named "conv2d", and a pointwise convolution also -// takes that name when its weights are prepacked. Those two are therefore -// separated by the weight's spatial extent. Shared by the global and local -// workgroup size functions below so that the two cannot disagree about the -// same dispatch. -Conv2dMethod infer_conv2d_method_from_shader( - ComputeGraph* graph, - const vkapi::ShaderInfo& shader, - const ValueRef weight_data) { - const std::string& kernel_name = shader.kernel_name; - // Checked before the plain "conv2d" test below, which "conv2d_dw" and - // "conv2d_pw" would otherwise match too. - if (kernel_name.find("conv2d_dw") != std::string::npos) { - return Conv2dMethod::Depthwise; - } - if (kernel_name.find("conv2d_pw") != std::string::npos) { - return Conv2dMethod::Pointwise; - } - if (kernel_name.find("conv_transpose2d") != std::string::npos) { - return Conv2dMethod::Transposed; - } - const auto& weight_sizes = graph->get_tref(weight_data)->sizes; - if (weight_sizes.at(2) == 1 && weight_sizes.at(3) == 1) { - return Conv2dMethod::Pointwise; - } - return Conv2dMethod::SlidingWindow; -} - -// Custom global workgroup size function for conv2d -GlobalWorkGrid conv2d_gwg( - ComputeGraph* graph, - const vkapi::ShaderInfo& shader, - const std::vector& args, - const std::vector& resize_args) { - const ValueRef out = args.at(0).refs.at(0); - const ValueRef weight_data = resize_args.at(0); - - const Conv2dMethod method = - infer_conv2d_method_from_shader(graph, shader, weight_data); - - // Determine stride_equals_dilation from shader name - bool stride_equals_dilation = - shader.kernel_name.find("_sned") == std::string::npos; - - const GlobalWorkGrid wg_size = create_conv2d_gwg( - *graph, method, out, weight_data, stride_equals_dilation); - - if (method == Conv2dMethod::Pointwise) { - utils::uvec3 pointwise_wg_size = {wg_size[0] * wg_size[1], wg_size[2], 1u}; - - if (shader.kernel_name.find("s1p0") != std::string::npos) { - pointwise_wg_size[0] *= 4; - } - return GlobalWorkGrid(pointwise_wg_size, kTiledWorkGrid); - } - - return wg_size; -} - -// Custom local workgroup size function for conv2d -LocalWorkGroup conv2d_lwg( - ComputeGraph* graph, - const vkapi::ShaderInfo& shader, - const GlobalWorkGrid& gwg, - const std::vector& args, - const std::vector& resize_args) { - (void)args; - - const ValueRef weight_data = resize_args.at(0); - const Conv2dMethod method = - infer_conv2d_method_from_shader(graph, shader, weight_data); - - if (method == Conv2dMethod::Pointwise) { - uint32_t lwg_y = 1; - if (gwg[1] % 8 == 0) { - lwg_y = 8; - } else if (gwg[1] % 4 == 0) { - lwg_y = 4; - } else if (gwg[1] % 2 == 0) { - lwg_y = 2; - } - return LocalWorkGroup(64u / lwg_y, lwg_y, 1u); - } else { - return graph->create_lwg(gwg); - } -} - -// Custom global workgroup size function for conv1d -GlobalWorkGrid conv1d_gwg( - ComputeGraph* graph, - const vkapi::ShaderInfo& shader, - const std::vector& args, - const std::vector& resize_args) { - (void)shader; - (void)resize_args; - const ValueRef out = args.at(0).refs.at(0); - - return GlobalWorkGrid( - {// out length - graph->size_at(-1, out), - // out channels - static_cast(graph->size_at(-2, out)), - // out batches - utils::div_up_4(graph->size_at(-3, out))}, - kTiledWorkGrid); -} - -void add_conv2d_node( - ComputeGraph& graph, - const ValueRef in, - const ValueRef weight_data, - const ValueRef bias, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef transposed, - const ValueRef output_padding, - const ValueRef groups, - const ValueRef out_min, - const ValueRef out_max, - const ValueRef out, - const bool clamp_out, - const bool force_direct) { - const bool transposed_val = graph.get_bool(transposed); - - float out_min_val = 0.0f; - float out_max_val = 0.0f; - if (out_min != kDummyValueRef) { - out_min_val = graph.extract_scalar(out_min); - } - if (out_max != kDummyValueRef) { - out_max_val = graph.extract_scalar(out_max); - } - - const int64_t groups_val = graph.get_int(groups); - - const Conv2dMethod method = - get_conv2d_method(graph, weight_data, groups_val, transposed_val); - - // Use tiled path for all pointwise conv2d - if (method == Conv2dMethod::Pointwise) { - return conv2d_pw_impl( - graph, - in, - weight_data, - bias, - stride, - padding, - out, - transposed_val, - clamp_out, - out_min_val, - out_max_val); - } - - if (method == Conv2dMethod::Depthwise) { - return conv2d_dw_impl( - graph, - in, - weight_data, - bias, - stride, - padding, - dilation, - out, - clamp_out, - out_min_val, - out_max_val); - } - - const Kernel2dParams kernel_params = create_kernel2d_params( - graph, - weight_data, - /*kernel_size_only = */ false, - stride, - padding, - dilation); - - // SlidingWindow conv2d: route to the im2col + GEMM path when the heuristic - // indicates it is beneficial, falling back to the direct convolution shader - // otherwise. `force_direct` bypasses the heuristic entirely and forces the - // direct path (used by tests to exercise the direct shader regardless of - // device); the default (false) reproduces the production routing exactly. - const bool use_im2col = !force_direct && - method == Conv2dMethod::SlidingWindow && - should_use_conv2d_im2col( - graph, weight_data, groups_val, kernel_params, out); - if (use_im2col) { - return conv2d_gemm_impl( - graph, - in, - weight_data, - bias, - stride, - padding, - dilation, - out, - clamp_out, - out_min_val, - out_max_val); - } - - ValueRef arg_weight = prepack_weights(graph, weight_data, method); - ValueRef arg_bias = prepack_biases( - graph, - bias, - weight_data, - transposed_val, - /* storage_type = */ utils::kTexture2D, - /* memory_layout = */ utils::kWidthPacked); - - const std::vector in_sizes = graph.sizes_of(in); - if (in_sizes.at(0) > 1) { - VK_THROW("conv2d: input batch size > 1 is not supported yet!"); - } - - check_conv_args(graph, in, out); - - Conv2dParams extra_params = - create_conv2d_params(graph, weight_data, kernel_params, transposed_val); - - const bool stride_equals_dilation = - (kernel_params.stride[0] == kernel_params.dilation[0] && - kernel_params.stride[1] == kernel_params.dilation[1]); - - const bool stride_1_padding_0 = - (kernel_params.stride[0] == 1 && kernel_params.stride[1] == 1 && - kernel_params.padding[0] == 0 && kernel_params.padding[1] == 0); - - OutputParams out_params = {out_min_val, out_max_val}; - - check_conv2d_params(kernel_params, transposed_val); - - vkapi::ShaderInfo shader = get_conv2d_shader( - graph, - out, - /*prepack_weights = */ false, - method, - weight_data, - clamp_out, - stride_equals_dilation, - stride_1_padding_0); - - vkapi::ParamsBindList param_buffers = { - graph.logical_limits_ubo(out), - graph.sizes_ubo(in), - graph.create_params_buffer(kernel_params), - graph.create_params_buffer(extra_params), - graph.create_params_buffer(out_params), - }; - - graph.execute_nodes().emplace_back(new DynamicDispatchNode( - graph, - shader, - conv2d_gwg, - conv2d_lwg, - // Inputs and Outputs - {{out, vkapi::kWrite}, {{in, arg_weight, arg_bias}, vkapi::kRead}}, - // Shader params buffers - param_buffers, - // Push Constants - {}, - // Specialization Constants - {utils::safe_downcast(groups_val)}, - // Resize Args - {weight_data, stride, padding, dilation, transposed, output_padding}, - // Resizing Logic - resize_conv2d_node)); -} - -void add_conv1d_node( - ComputeGraph& graph, - const ValueRef in, - const ValueRef weight, - const ValueRef bias, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef groups, - const ValueRef out_min, - const ValueRef out_max, - const ValueRef out, - const bool clamp_out) { - ValueRef arg_weight = prepack_standard( - graph, - weight, - graph.storage_type_of(out), - utils::kChannelsPacked, - /* passthrough = */ false); - ValueRef arg_bias = prepack_biases( - graph, - bias, - weight, - /*transposed = */ false, - /*storage_type = */ utils::kTexture3D, - /*memory_layout = */ utils::kWidthPacked); - - float out_min_val = 0.0f; - float out_max_val = 0.0f; - if (out_min != kDummyValueRef) { - out_min_val = graph.extract_scalar(out_min); - } - if (out_max != kDummyValueRef) { - out_max_val = graph.extract_scalar(out_max); - } - - const int64_t groups_val = graph.get_int(groups); - - const std::vector in_sizes = graph.sizes_of(in); - const std::vector weight_sizes = graph.sizes_of(arg_weight); - const std::vector out_sizes = graph.sizes_of(out); - - check_conv_args(graph, in, out); - - const int32_t in_channels = in_sizes.at(1); - const int32_t out_channels = weight_sizes.at(0); - const int32_t kernel_size = weight_sizes.at(2); - const int32_t stride_size = graph.get_int_list(stride)->at(0); - const int32_t padding_size = graph.get_int_list(padding)->at(0); - const int32_t dilation_size = graph.get_int_list(dilation)->at(0); - const int32_t in_group_size = static_cast(in_channels / groups_val); - const int32_t out_group_size = - static_cast(out_channels / groups_val); - - Kernel1dParams kernel_params = { - kernel_size, - stride_size, - padding_size, - dilation_size, - in_group_size, - out_group_size}; - - const OutputParams out_params = {out_min_val, out_max_val}; - - std::string kernel_name("conv1d"); - if (clamp_out) { - kernel_name += "_clamp"; - } - kernel_name.reserve(kShaderNameReserve); - - add_dtype_suffix(kernel_name, graph.dtype_of(out)); - - graph.execute_nodes().emplace_back(new DynamicDispatchNode( - graph, - VK_KERNEL_FROM_STR(kernel_name), - conv1d_gwg, - default_pick_lwg, - // Inputs and Outputs - {{out, vkapi::kWrite}, {{in, arg_weight, arg_bias}, vkapi::kRead}}, - // Shader params buffers - { - graph.logical_limits_ubo(out), - graph.sizes_ubo(in), - graph.create_params_buffer(kernel_params), - graph.create_params_buffer(out_params), - }, - // Push Constants - {}, - // Specialization Constants - {graph.hashed_layout_of(out), - graph.hashed_layout_of(in), - graph.hashed_layout_of(arg_weight), - graph.hashed_layout_of(arg_bias)}, - // Resize Args - {weight, stride, padding, dilation}, - // Resizing Logic - resize_conv1d_node)); -} - -void conv(ComputeGraph& graph, const std::vector& args) { - int64_t in_ndim = graph.dim_of(args[0]); - if (in_ndim == 4) { - if (args.size() == 10) { - // ordinary conv2d - return add_conv2d_node( - graph, - args[0], - args[1], - args[2], - args[3], - args[4], - args[5], - args[6], - args[7], - args[8], - /*out_min = */ kDummyValueRef, - /*out_max = */ kDummyValueRef, - args[9], - false); - } else { - // conv2d with clamp - return add_conv2d_node( - graph, - args[0], - args[1], - args[2], - args[3], - args[4], - args[5], - args[6], - args[7], - args[8], - args[9], - args[10], - args[11], - true); - } - } else { - // Conv1d path - if (graph.packed_dim_of(args[0]) == WHCN::kHeightDim) { - // Height-packed: route to optimized conv1d implementations - const auto weight_sizes = graph.sizes_of(args[1]); - const int64_t groups_val = graph.get_int(args[8]); - const bool is_pointwise = weight_sizes.at(2) == 1; - const bool is_depthwise = - groups_val == weight_sizes.at(0) && weight_sizes.at(1) == 1; - - // Build unified 10-arg vector: - // in, weight, bias, stride, padding, dilation, groups, - // output_min, output_max, out - // For non-clamp (args.size() == 10): output_min/max = kDummyValueRef - // For clamp (args.size() == 12): output_min/max from args[9]/args[10] - ValueRef output_min = kDummyValueRef; - ValueRef output_max = kDummyValueRef; - ValueRef out; - if (args.size() == 10) { - out = args[9]; - } else { - output_min = args[9]; - output_max = args[10]; - out = args[11]; - } - - std::vector conv1d_args = { - args[0], - args[1], - args[2], - args[3], - args[4], - args[5], - args[8], - output_min, - output_max, - out}; - - if (is_pointwise) { - VK_GET_OP_FN("et_vk.conv1d_pw.default")(graph, conv1d_args); - } else if (is_depthwise) { - VK_GET_OP_FN("et_vk.conv1d_dw.default")(graph, conv1d_args); - } else { - VK_THROW( - "Height-packed conv1d only supports pointwise (K=1) or " - "depthwise (groups=C)"); - } - return; - } - - // Existing channels-packed fallback - if (args.size() == 10) { - // ordinary conv1d - return add_conv1d_node( - graph, - args[0], - args[1], - args[2], - args[3], - args[4], - args[5], - args[8], - /*out_min = */ kDummyValueRef, - /*out_max = */ kDummyValueRef, - args[9], - false); - } else { - // conv1d with clamp - return add_conv1d_node( - graph, - args[0], - args[1], - args[2], - args[3], - args[4], - args[5], - args[8], - args[9], - args[10], - args[11], - true); - } - } -} - -REGISTER_OPERATORS { - VK_REGISTER_OP(aten.convolution.default, conv); - VK_REGISTER_OP(conv_with_clamp.default, conv); - VK_REGISTER_OP(et_vk.conv_with_clamp.default, conv); -} - -} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/EmbeddingQ4gsw.cpp b/backends/vulkan/runtime/graph/ops/impl/EmbeddingQ4gsw.cpp index ad4786d1c60..2d0d8c623f4 100644 --- a/backends/vulkan/runtime/graph/ops/impl/EmbeddingQ4gsw.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/EmbeddingQ4gsw.cpp @@ -9,8 +9,8 @@ #include #include -#include #include +#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/GemmCommon.h b/backends/vulkan/runtime/graph/ops/impl/GemmCommon.h deleted file mode 100644 index eadeeee7080..00000000000 --- a/backends/vulkan/runtime/graph/ops/impl/GemmCommon.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include -#include - -namespace vkcompute { - -// Helpers shared by the linear/matmul tiled and coopmat dispatch paths. - -// Prepack a floating-point weight tensor for the linear/matmul kernels. -// Source layout: [N, K] (is_transposed=true) or [K, N] (is_transposed=false), -// optionally batched. Output layout: 4OC x 4IC blocked, packed as -// kWidthPacked. When force_buffer is true, the packed tensor uses buffer -// storage (required by the coopmat shader); otherwise texture2d is used -// when it fits within max_texture2d_dim. -ValueRef prepack_fp_linear_weight( - ComputeGraph& graph, - const ValueRef weight_data, - bool is_transposed, - int64_t B, - bool force_buffer = false); - -// Resize logic for linear-shaped output: takes M from the input's penultimate -// dim, N from resize_args[0], optional batch from input's leading dim. -void resize_linear_node( - ComputeGraph* graph, - const std::vector& args, - const std::vector& resize_args); - -// Resize logic for matmul (mat1 @ mat2): output rows from mat1, cols from -// mat2, batch dims propagated from mat1. -void resize_matmul_tiled_node( - ComputeGraph* graph, - const std::vector& args, - const std::vector& resize_args); - -} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.h b/backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.h deleted file mode 100644 index e48d1e32a08..00000000000 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.h +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include -#include -#include - -namespace vkcompute { - -inline constexpr int64_t kQ8taConv2dIm2ColScratchBudgetBytes = 16 * 1024 * 1024; -inline constexpr int64_t kQ8taConv2dMaxRowsPerTile = 65535; - -struct Q8taConv2dStreamPlan final { - int64_t aligned_out_width; - int64_t rows_per_tile; - int64_t num_tiles; - int64_t scratch_bytes; - bool feasible; -}; - -Q8taConv2dStreamPlan make_q8ta_conv2d_stream_plan( - int64_t batch, - int64_t flattened_kernel_size, - int64_t out_height, - int64_t out_width, - int64_t scratch_budget_bytes); - -// max_buffer_bytes is Adapter::max_buffer_numel(), which returns -// maxStorageBufferRange in bytes (not elements) — directly comparable with -// the byte-denominated scratch budget. -Q8taConv2dStreamPlan make_q8ta_conv2d_stream_plan_for_device( - int64_t batch, - int64_t flattened_kernel_size, - int64_t out_height, - int64_t out_width, - uint64_t max_buffer_bytes); - -enum class ActivationType : uint32_t { - kNone = 0, - kRelu = 1, -}; - -ActivationType activation_type_from_string(const std::string& activation); - -bool q8ta_conv2d_check_packed_dim_info(const api::PackedDimInfo& info); - -bool q8ta_conv2d_check_4w4c_packed_dim_info(const api::PackedDimInfo& info); - -ValueRef prepack_quantized_conv2d_weight( - ComputeGraph& graph, - const QuantizationConfig& weight_quant_config, - const ValueRef weight_data, - const ValueRef input, - const ValueRef output, - const ValueRef groups, - const ValueRef kernel_size); - -ValueRef prepack_quantized_conv2d_weight( - ComputeGraph& graph, - const QuantizationConfig& weight_quant_config, - const ValueRef weight_data, - const ValueRef input, - const ValueRef output, - const ValueRef groups, - const ValueRef kernel_size); - -ValueRef prepack_quantized_conv2d_dw_weight( - ComputeGraph& graph, - const QuantizationConfig& weight_quant_config, - const ValueRef weight_data, - const ValueRef kernel_size); - -void add_q8ta_conv2d_dw_node( - ComputeGraph& graph, - const ValueRef packed_int8_input, - const ValueRef input_scale, - const ValueRef input_zp, - const ValueRef packed_weight, - const ValueRef packed_weight_sums, - const ValueRef packed_weight_scales, - const ValueRef output_scale, - const ValueRef output_zp, - const ValueRef bias_data, - const ValueRef packed_bias, - const ValueRef kernel_size, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef groups, - const uint32_t activation_type, - const ValueRef packed_int8_output); - -void add_conv2d_dw_q8ta_q8csw_q8to_4w4c_node( - ComputeGraph& graph, - const ValueRef packed_int8_input, - const ValueRef input_scale, - const ValueRef input_zp, - const ValueRef packed_weight, - const ValueRef packed_weight_sums, - const ValueRef packed_weight_scales, - const ValueRef output_scale, - const ValueRef output_zp, - const ValueRef bias_data, - const ValueRef packed_bias, - const ValueRef kernel_size, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef groups, - const ValueRef packed_int8_output); - -void add_q8ta_conv2d_node( - ComputeGraph& graph, - const ValueRef packed_int8_input, - const ValueRef packed_int8_input_im2col, - const ValueRef input_scale, - const ValueRef input_zp, - const ValueRef packed_weight, - const ValueRef packed_weight_sums, - const ValueRef packed_weight_scales, - const ValueRef output_scale, - const ValueRef output_zp, - const ValueRef bias_data, - const ValueRef packed_bias, - const ValueRef kernel_size, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef groups, - const uint32_t activation_type, - const ValueRef packed_int8_output); - -void add_q8ta_conv2d_pw_node( - ComputeGraph& graph, - const bool use_unsigned_dot, - const ValueRef packed_int8_input, - const ValueRef input_scale, - const ValueRef input_zp, - const ValueRef packed_weight, - const ValueRef packed_weight_sums, - const ValueRef packed_weight_scales, - const ValueRef output_scale, - const ValueRef output_zp, - const ValueRef bias_data, - const ValueRef packed_bias, - const uint32_t activation_type, - const ValueRef packed_int8_output, - const int32_t groups = 1, - const ValueRef conv_input = kDummyValueRef, - const ValueRef kernel_size = kDummyValueRef, - const ValueRef stride = kDummyValueRef, - const ValueRef padding = kDummyValueRef, - const ValueRef dilation = kDummyValueRef, - const bool is_im2col = false, - const ValueRef stream_row_offset_ref = kDummyValueRef, - const ValueRef max_im2col_rows_ref = kDummyValueRef); - -constexpr int64_t kMaxUnsignedDotAccumulatorBytes = 33025; - -bool can_use_unsigned_pw_dot( - const vkapi::Adapter& adapter, - int64_t k_per_group); - -void q8ta_conv2d_pw_impl( - ComputeGraph& graph, - bool use_unsigned_dot, - const std::vector& args); - -void q8ta_conv2d_pw(ComputeGraph& graph, const std::vector& args); - -std::vector calculate_q8ta_im2col_sizes( - ComputeGraph* graph, - const ValueRef& input, - const ValueRef& output, - const ValueRef& kernel_size, - const ValueRef& groups); - -void add_q8ta_im2col_node( - ComputeGraph& graph, - const ValueRef packed_int8_input, - const ValueRef kernel_size, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef groups, - const ValueRef packed_int8_output, - const ValueRef packed_int8_im2col, - const int32_t zp, - const ValueRef stream_row_offset_ref, - const ValueRef max_im2col_rows_ref = kDummyValueRef); - -void q8ta_conv2d_im2col(ComputeGraph& graph, const std::vector& args); - -void q8ta_conv2d_im2col_impl( - ComputeGraph& graph, - bool use_unsigned_dot, - const std::vector& args); - -void q8ta_conv2d_general( - ComputeGraph& graph, - const std::vector& args); - -// Transposed convolution - -void q8ta_conv2d_transposed( - ComputeGraph& graph, - const std::vector& args); - -} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.h b/backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.h deleted file mode 100644 index 4ab87f8ac33..00000000000 --- a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include -#include -#include - -namespace vkcompute { - -LocalWorkGroup quantized_linear_lwg( - ComputeGraph* graph, - const vkapi::ShaderInfo& shader, - const GlobalWorkGrid& gwg, - const std::vector& args, - const std::vector& resize_args); - -ValueRef prepack_quantized_linear_weight( - ComputeGraph& graph, - const QuantizationConfig& weight_quant_config, - const ValueRef qmat2_data, - const bool use_unsigned_dot = false); - -} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/SDPA.cpp b/backends/vulkan/runtime/graph/ops/impl/SDPA.cpp index 39854b23d02..bc315727bd2 100644 --- a/backends/vulkan/runtime/graph/ops/impl/SDPA.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/SDPA.cpp @@ -12,10 +12,9 @@ #include #include -#include #include #include -#include +#include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/BinaryOp.cpp b/backends/vulkan/runtime/graph/ops/impl/binary/BinaryOp.cpp similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/BinaryOp.cpp rename to backends/vulkan/runtime/graph/ops/impl/binary/BinaryOp.cpp diff --git a/backends/vulkan/runtime/graph/ops/impl/BinaryScalarOp.cpp b/backends/vulkan/runtime/graph/ops/impl/binary/BinaryScalarOp.cpp similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/BinaryScalarOp.cpp rename to backends/vulkan/runtime/graph/ops/impl/binary/BinaryScalarOp.cpp diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taBinary.cpp b/backends/vulkan/runtime/graph/ops/impl/binary/q8ta/Q8taBinary.cpp similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Q8taBinary.cpp rename to backends/vulkan/runtime/graph/ops/impl/binary/q8ta/Q8taBinary.cpp diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taBinary.h b/backends/vulkan/runtime/graph/ops/impl/binary/q8ta/Q8taBinary.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Q8taBinary.h rename to backends/vulkan/runtime/graph/ops/impl/binary/q8ta/Q8taBinary.h diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.cpp new file mode 100644 index 00000000000..4bbb2c56911 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.cpp @@ -0,0 +1,84 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include + +#include +#include + +#include + +namespace vkcompute { + +ValueRef prepack_biases( + ComputeGraph& graph, + const ValueRef vref, + const ValueRef weight, + const bool transposed, + const utils::StorageType storage_type, + const utils::GPUMemoryLayout memory_layout) { + auto sizes = graph.sizes_of(weight); + const int64_t out_channels = transposed ? sizes.at(1) : sizes.at(0); + + ValueRef v = graph.add_tensor( + {out_channels}, graph.dtype_of(weight), storage_type, memory_layout); + + vkapi::ShaderInfo shader = + get_nchw_to_tensor_shader(graph, v, graph.get_staging_dtype_for(weight)); + + vkapi::ParamsBindList param_buffers = {}; + if (graph.is_buffer_storage(v)) { + param_buffers.append(graph.buffer_meta_ubo(v)); + } else { + param_buffers.append(graph.texture_meta_ubo(v)); + } + + std::vector pcs; + if (graph.is_buffer_storage(v)) { + pcs = {graph.sizes_pc_of(v), graph.strides_pc_of(v), graph.numel_pc_of(v)}; + } + + graph.prepack_nodes().emplace_back(new PrepackNode( + graph, + shader, + graph.create_gwg(v), + graph.create_lwg(v), + vref, + v, + param_buffers, + // Specialization constants + {graph.hashed_layout_of(v)}, + pcs)); + + return v; +} + +void check_conv_args( + ComputeGraph& graph, + const ValueRef in, + const ValueRef out) { + VK_CHECK_COND(graph.packed_dim_of(in) == WHCN::kChannelsDim); + VK_CHECK_COND(graph.packed_dim_of(out) == WHCN::kChannelsDim); +} + +void conv(ComputeGraph& graph, const std::vector& args) { + if (graph.dim_of(args[0]) == 4) { + return conv2d_impl(graph, args); + } + return conv1d_impl(graph, args); +} + +REGISTER_OPERATORS { + VK_REGISTER_OP(aten.convolution.default, conv); + VK_REGISTER_OP(conv_with_clamp.default, conv); + VK_REGISTER_OP(et_vk.conv_with_clamp.default, conv); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.h b/backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.h new file mode 100644 index 00000000000..08bf5bd5c9c --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/Convolution.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +ValueRef prepack_biases( + ComputeGraph& graph, + const ValueRef vref, + const ValueRef weight, + const bool transposed, + const utils::StorageType storage_type, + const utils::GPUMemoryLayout memory_layout); + +void check_conv_args( + ComputeGraph& graph, + const ValueRef in, + const ValueRef out); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.cpp new file mode 100644 index 00000000000..1f71e009d32 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.cpp @@ -0,0 +1,100 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include + +#include + +namespace vkcompute { + +void conv1d_impl(ComputeGraph& graph, const std::vector& args) { + if (graph.packed_dim_of(args[0]) == WHCN::kHeightDim) { + // Height-packed: route to optimized conv1d implementations + const auto weight_sizes = graph.sizes_of(args[1]); + const int64_t groups_val = graph.get_int(args[8]); + const bool is_pointwise = weight_sizes.at(2) == 1; + const bool is_depthwise = + groups_val == weight_sizes.at(0) && weight_sizes.at(1) == 1; + + // Build unified 10-arg vector: + // in, weight, bias, stride, padding, dilation, groups, + // output_min, output_max, out + // For non-clamp (args.size() == 10): output_min/max = kDummyValueRef + // For clamp (args.size() == 12): output_min/max from args[9]/args[10] + ValueRef output_min = kDummyValueRef; + ValueRef output_max = kDummyValueRef; + ValueRef out; + if (args.size() == 10) { + out = args[9]; + } else { + output_min = args[9]; + output_max = args[10]; + out = args[11]; + } + + std::vector conv1d_args = { + args[0], + args[1], + args[2], + args[3], + args[4], + args[5], + args[8], + output_min, + output_max, + out}; + + if (is_pointwise) { + VK_GET_OP_FN("et_vk.conv1d_pw.default")(graph, conv1d_args); + } else if (is_depthwise) { + VK_GET_OP_FN("et_vk.conv1d_dw.default")(graph, conv1d_args); + } else { + VK_THROW( + "Height-packed conv1d only supports pointwise (K=1) or " + "depthwise (groups=C)"); + } + return; + } + + // Existing channels-packed fallback + if (args.size() == 10) { + // ordinary conv1d + return add_conv1d_direct_node( + graph, + args[0], + args[1], + args[2], + args[3], + args[4], + args[5], + args[8], + /*out_min = */ kDummyValueRef, + /*out_max = */ kDummyValueRef, + args[9], + false); + } else { + // conv1d with clamp + return add_conv1d_direct_node( + graph, + args[0], + args[1], + args[2], + args[3], + args[4], + args[5], + args[8], + args[9], + args[10], + args[11], + true); + } +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.h new file mode 100644 index 00000000000..f6030128587 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1d.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +void conv1d_impl(ComputeGraph& graph, const std::vector& args); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv1dDW.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDW.cpp similarity index 97% rename from backends/vulkan/runtime/graph/ops/impl/Conv1dDW.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDW.cpp index ba59090c124..58971984821 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Conv1dDW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDW.cpp @@ -150,7 +150,7 @@ void add_conv1d_dw_node( // Args: in, weight, bias, stride, padding, dilation, groups, // output_min, output_max, out // output_min and output_max may be kDummyValueRef (no clamp). -void conv1d_dw(ComputeGraph& graph, const std::vector& args) { +void conv1d_dw_impl(ComputeGraph& graph, const std::vector& args) { ValueRef in = args[0]; ValueRef weight = args[1]; ValueRef bias = args[2]; @@ -182,7 +182,7 @@ void conv1d_dw(ComputeGraph& graph, const std::vector& args) { } REGISTER_OPERATORS { - VK_REGISTER_OP(et_vk.conv1d_dw.default, conv1d_dw); + VK_REGISTER_OP(et_vk.conv1d_dw.default, conv1d_dw_impl); } } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.cpp new file mode 100644 index 00000000000..bdd6f8da6e9 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.cpp @@ -0,0 +1,179 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include +#include + +#include + +#include + +namespace vkcompute { + +void resize_conv1d_node( + ComputeGraph* graph, + const std::vector& args, + const std::vector& extra_args) { + const ValueRef out = args.at(0).refs.at(0); + const ValueRef self = args.at(1).refs.at(0); + TensorRefPtr weight_ref = graph->get_tref(extra_args.at(0)); + + const int64_t stride_size = graph->get_int_list(extra_args.at(1))->at(0); + const int64_t padding_size = graph->get_int_list(extra_args.at(2))->at(0); + const int64_t dilation_size = graph->get_int_list(extra_args.at(3))->at(0); + + const std::vector& weight_sizes = weight_ref->sizes; + + const std::vector in_sizes = graph->sizes_of(self); + const size_t ndim = in_sizes.size(); + std::vector new_out_sizes(ndim); + + const int64_t kernel_size = weight_sizes.at(2); + const int64_t in_length = in_sizes.at(2); + + new_out_sizes.at(0) = in_sizes.at(0); + new_out_sizes.at(1) = weight_sizes.at(0); + new_out_sizes.at(2) = calc_out_size( + in_length, kernel_size, stride_size, padding_size, dilation_size, false); + + graph->virtual_resize(out, new_out_sizes); +} + +struct OutputParams final { + float out_min; + float out_max; +}; + +// Custom global workgroup size function for conv1d +GlobalWorkGrid conv1d_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + (void)shader; + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + + return GlobalWorkGrid( + {// out length + graph->size_at(-1, out), + // out channels + static_cast(graph->size_at(-2, out)), + // out batches + utils::div_up_4(graph->size_at(-3, out))}, + kTiledWorkGrid); +} + +void add_conv1d_direct_node( + ComputeGraph& graph, + const ValueRef in, + const ValueRef weight, + const ValueRef bias, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef groups, + const ValueRef out_min, + const ValueRef out_max, + const ValueRef out, + const bool clamp_out) { + ValueRef arg_weight = prepack_standard( + graph, + weight, + graph.storage_type_of(out), + utils::kChannelsPacked, + /* passthrough = */ false); + ValueRef arg_bias = prepack_biases( + graph, + bias, + weight, + /*transposed = */ false, + /*storage_type = */ utils::kTexture3D, + /*memory_layout = */ utils::kWidthPacked); + + float out_min_val = 0.0f; + float out_max_val = 0.0f; + if (out_min != kDummyValueRef) { + out_min_val = graph.extract_scalar(out_min); + } + if (out_max != kDummyValueRef) { + out_max_val = graph.extract_scalar(out_max); + } + + const int64_t groups_val = graph.get_int(groups); + + const std::vector in_sizes = graph.sizes_of(in); + const std::vector weight_sizes = graph.sizes_of(arg_weight); + const std::vector out_sizes = graph.sizes_of(out); + + check_conv_args(graph, in, out); + + const int32_t in_channels = utils::safe_downcast(in_sizes.at(1)); + const int32_t out_channels = + utils::safe_downcast(weight_sizes.at(0)); + const int32_t kernel_size = utils::safe_downcast(weight_sizes.at(2)); + const int32_t stride_size = + utils::safe_downcast(graph.get_int_list(stride)->at(0)); + const int32_t padding_size = + utils::safe_downcast(graph.get_int_list(padding)->at(0)); + const int32_t dilation_size = + utils::safe_downcast(graph.get_int_list(dilation)->at(0)); + const int32_t in_group_size = + utils::safe_downcast(in_channels / groups_val); + const int32_t out_group_size = + utils::safe_downcast(out_channels / groups_val); + + Kernel1dParams kernel_params = { + kernel_size, + stride_size, + padding_size, + dilation_size, + in_group_size, + out_group_size}; + + const OutputParams out_params = {out_min_val, out_max_val}; + + std::string kernel_name("conv1d"); + if (clamp_out) { + kernel_name += "_clamp"; + } + kernel_name.reserve(kShaderNameReserve); + + add_dtype_suffix(kernel_name, graph.dtype_of(out)); + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + VK_KERNEL_FROM_STR(kernel_name), + conv1d_gwg, + default_pick_lwg, + // Inputs and Outputs + {{out, vkapi::kWrite}, {{in, arg_weight, arg_bias}, vkapi::kRead}}, + // Shader params buffers + { + graph.logical_limits_ubo(out), + graph.sizes_ubo(in), + graph.create_params_buffer(kernel_params), + graph.create_params_buffer(out_params), + }, + // Push Constants + {}, + // Specialization Constants + {graph.hashed_layout_of(out), + graph.hashed_layout_of(in), + graph.hashed_layout_of(arg_weight), + graph.hashed_layout_of(arg_bias)}, + // Resize Args + {weight, stride, padding, dilation}, + // Resizing Logic + resize_conv1d_node)); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.h new file mode 100644 index 00000000000..9800330b9f8 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dDirect.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +void add_conv1d_direct_node( + ComputeGraph& graph, + const ValueRef in, + const ValueRef weight, + const ValueRef bias, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef groups, + const ValueRef out_min, + const ValueRef out_max, + const ValueRef out, + const bool clamp_out); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv1dPW.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dPW.cpp similarity index 97% rename from backends/vulkan/runtime/graph/ops/impl/Conv1dPW.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dPW.cpp index de2fcdbb806..22457c37e24 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Conv1dPW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv1d/Conv1dPW.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include @@ -211,7 +210,7 @@ void add_conv1d_pw_node( // Args: in, weight, bias, stride, padding, dilation, groups, // output_min, output_max, out // output_min and output_max may be kDummyValueRef (no clamp). -void conv1d_pw(ComputeGraph& graph, const std::vector& args) { +void conv1d_pw_impl(ComputeGraph& graph, const std::vector& args) { ValueRef in = args[0]; ValueRef weight = args[1]; ValueRef bias = args[2]; @@ -236,7 +235,7 @@ void conv1d_pw(ComputeGraph& graph, const std::vector& args) { } REGISTER_OPERATORS { - VK_REGISTER_OP(et_vk.conv1d_pw.default, conv1d_pw); + VK_REGISTER_OP(et_vk.conv1d_pw.default, conv1d_pw_impl); } } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2d.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2d.cpp new file mode 100644 index 00000000000..ff4f8173780 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2d.cpp @@ -0,0 +1,234 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include +#include + +#include + +namespace vkcompute { + +void resize_conv2d_node( + ComputeGraph* graph, + const std::vector& args, + const std::vector& extra_args) { + const ValueRef out = args.at(0).refs.at(0); + const ValueRef self = args.at(1).refs.at(0); + + size_t ndim = graph->dim_of(self); + std::vector new_out_sizes(ndim); + const bool transposed = graph->get_bool(extra_args.at(4)); + + std::vector self_sizes = graph->sizes_of(self); + // Batch, Channel + if (ndim == 4) { + new_out_sizes.at(ndim - 4) = self_sizes.at(ndim - 4); + } + + TensorRefPtr weight_ref = graph->get_tref(extra_args.at(0)); + const auto& weight_sizes = weight_ref->sizes; + new_out_sizes.at(ndim - 3) = + transposed ? weight_sizes.at(ndim - 3) : weight_sizes.at(ndim - 4); + + // Height, Width + const auto& new_out_sizes_hw = calc_out_sizes_hw( + *graph, + self_sizes, + extra_args.at(0), + /*kernel_size_only = */ false, + {extra_args.at(1), extra_args.at(2), extra_args.at(3), extra_args.at(5)}, + transposed); + new_out_sizes.at(ndim - 2) = new_out_sizes_hw.at(0); + new_out_sizes.at(ndim - 1) = new_out_sizes_hw.at(1); + + graph->virtual_resize(out, new_out_sizes); +} + +Conv2dMethod get_conv2d_method( + ComputeGraph& graph, + const ValueRef weight, + const int64_t groups, + const bool transposed) { + const auto weight_sizes = graph.sizes_of(weight); + if (!transposed && weight_sizes.at(0) == groups && weight_sizes.at(1) == 1) { + return Conv2dMethod::Depthwise; + } + if (transposed) { + return Conv2dMethod::Transposed; + } + if (weight_sizes.at(2) == 1 && weight_sizes.at(3) == 1) { + return Conv2dMethod::Pointwise; + } + return Conv2dMethod::SlidingWindow; +} + +// Decide whether a SlidingWindow conv2d should be computed via the +// im2col + GEMM path (conv2d_gemm_impl) instead of the direct convolution +// shader. Across 26 configs on Mali-G715 (buffer path) and Adreno SM8650 +// (texture path): FP32 cases were numerically verified against the reference; +// FP16 cases were routing/dispatch-validated only (the reference is float-only +// for the large shapes, so FP16 outputs were not numerically checked). +// +// Only called for SlidingWindow conv2d (1x1 is routed to conv2d_pw and +// Depthwise/Transposed are handled before the call site). +// +// Preconditions (fall back to direct conv if any fail — the im2col path is +// either not applicable or not beneficial): +// - groups == 1 +// - dilation == 1 (all dims) +// +// Selection rule: use im2col once the output channel count is large enough to +// amortize the fixed ~N*K_total im2col gather cost. +constexpr int64_t kIm2colMinCOut = 128; + +// A cheap gather is a second, independent reason to take im2col. The gather +// materializes an N x K_total matrix before the GEMM runs, and a large gather +// is fine when c_out is large because the GEMM reads it back c_out times. When +// c_out is small it is only worth paying if the matrix is small outright, which +// is where the direct shader on Mali loses badly: an 80x80 3x3 conv wants ~4M +// elements, a 640x640 9x9 conv wants ~100M. +constexpr int64_t kIm2colMaxCheapGatherElements = 32 * 1024 * 1024; + +bool should_use_conv2d_im2col( + ComputeGraph& graph, + const ValueRef weight_data, + const int64_t groups_val, + const Kernel2dParams& kernel_params, + const ValueRef out) { + if (groups_val != 1) { + return false; + } + if (kernel_params.dilation[0] != 1 || kernel_params.dilation[1] != 1) { + return false; + } + const auto weight_sizes = graph.sizes_of(weight_data); + const int64_t c_out = weight_sizes.at(0); + const int64_t k_total = + weight_sizes.at(1) * weight_sizes.at(2) * weight_sizes.at(3); + + const auto out_sizes = graph.sizes_of(out); + const size_t ndim = out_sizes.size(); + const int64_t n = out_sizes.at(ndim - 1) * out_sizes.at(ndim - 2); + return c_out >= kIm2colMinCOut || + (graph.device_is_mali() && n * k_total <= kIm2colMaxCheapGatherElements); +} + +void conv2d_impl(ComputeGraph& graph, const std::vector& args) { + const ValueRef in = args[0]; + const ValueRef weight_data = args[1]; + const ValueRef bias = args[2]; + const ValueRef stride = args[3]; + const ValueRef padding = args[4]; + const ValueRef dilation = args[5]; + const ValueRef transposed = args[6]; + const ValueRef output_padding = args[7]; + const ValueRef groups = args[8]; + // conv2d with clamp passes out_min and out_max before the output + const bool clamp_out = args.size() != 10; + const ValueRef out_min = clamp_out ? args[9] : kDummyValueRef; + const ValueRef out_max = clamp_out ? args[10] : kDummyValueRef; + const ValueRef out = clamp_out ? args[11] : args[9]; + + const bool transposed_val = graph.get_bool(transposed); + + float out_min_val = 0.0f; + float out_max_val = 0.0f; + if (out_min != kDummyValueRef) { + out_min_val = graph.extract_scalar(out_min); + } + if (out_max != kDummyValueRef) { + out_max_val = graph.extract_scalar(out_max); + } + + const int64_t groups_val = graph.get_int(groups); + + const Conv2dMethod method = + get_conv2d_method(graph, weight_data, groups_val, transposed_val); + + // Use tiled path for all pointwise conv2d + if (method == Conv2dMethod::Pointwise) { + return conv2d_pw_impl( + graph, + in, + weight_data, + bias, + stride, + padding, + out, + transposed_val, + clamp_out, + out_min_val, + out_max_val); + } + + if (method == Conv2dMethod::Depthwise) { + return conv2d_dw_impl( + graph, + in, + weight_data, + bias, + stride, + padding, + dilation, + out, + clamp_out, + out_min_val, + out_max_val); + } + + const Kernel2dParams kernel_params = create_kernel2d_params( + graph, + weight_data, + /*kernel_size_only = */ false, + stride, + padding, + dilation); + + // SlidingWindow conv2d: route to the im2col + GEMM path when the heuristic + // indicates it is beneficial, falling back to the direct convolution shader + // otherwise. + const bool use_im2col = + method == Conv2dMethod::SlidingWindow && + should_use_conv2d_im2col( + graph, weight_data, groups_val, kernel_params, out); + if (use_im2col) { + return conv2d_gemm_impl( + graph, + in, + weight_data, + bias, + stride, + padding, + dilation, + out, + clamp_out, + out_min_val, + out_max_val); + } + + return conv2d_direct_impl( + graph, + in, + weight_data, + bias, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + out, + clamp_out, + out_min_val, + out_max_val); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Convolution.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2d.h similarity index 50% rename from backends/vulkan/runtime/graph/ops/impl/Convolution.h rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2d.h index 12e4733e67a..446859c363d 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Convolution.h +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2d.h @@ -12,18 +12,12 @@ namespace vkcompute { -ValueRef prepack_biases( - ComputeGraph& graph, - const ValueRef vref, - const ValueRef weight, - const bool transposed, - const utils::StorageType storage_type, - const utils::GPUMemoryLayout memory_layout); - -void check_conv_args( - ComputeGraph& graph, - const ValueRef in, - const ValueRef out); +enum class Conv2dMethod : uint8_t { + Depthwise, + Pointwise, + SlidingWindow, + Transposed, +}; void conv2d_pw_impl( ComputeGraph& graph, @@ -56,26 +50,6 @@ void resize_conv2d_node( const std::vector& args, const std::vector& extra_args); -// `force_direct` overrides the im2col-vs-direct routing heuristic: when true, -// a SlidingWindow conv2d always takes the direct sliding-window path, -// bypassing should_use_conv2d_im2col(). The default (false) preserves the -// production routing exactly. Pointwise / Depthwise / Transposed methods are -// unaffected by this flag. -void add_conv2d_node( - ComputeGraph& graph, - const ValueRef in, - const ValueRef weight_data, - const ValueRef bias, - const ValueRef stride, - const ValueRef padding, - const ValueRef dilation, - const ValueRef transposed, - const ValueRef output_padding, - const ValueRef groups, - const ValueRef out_min, - const ValueRef out_max, - const ValueRef out, - const bool clamp_out, - const bool force_direct = false); +void conv2d_impl(ComputeGraph& graph, const std::vector& args); } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv2dDW.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDW.cpp similarity index 98% rename from backends/vulkan/runtime/graph/ops/impl/Conv2dDW.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDW.cpp index a21f6c93e4e..0ac6a56d7ad 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Conv2dDW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDW.cpp @@ -6,10 +6,11 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include +#include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.cpp new file mode 100644 index 00000000000..a476d03a900 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.cpp @@ -0,0 +1,402 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include +#include + +#include + +#include + +namespace vkcompute { + +vkapi::ShaderInfo get_conv2d_shader( + ComputeGraph& graph, + const ValueRef out, + const bool prepack_weights, + const Conv2dMethod method, + const ValueRef weight, + const bool clamp_out = false, + const bool stride_equals_dilation = false, + const bool stride_1_padding_0 = false) { + (void)stride_equals_dilation; + std::string kernel_name; + kernel_name.reserve(kShaderNameReserve); + switch (method) { + case Conv2dMethod::Depthwise: + kernel_name = "conv2d_dw"; + break; + case Conv2dMethod::Pointwise: + if (prepack_weights) { + kernel_name = "conv2d"; + } else { + kernel_name = stride_1_padding_0 ? "conv2d_pw_s1p0" : "conv2d_pw"; + } + break; + case Conv2dMethod::SlidingWindow: + kernel_name = "conv2d"; + break; + case Conv2dMethod::Transposed: + kernel_name = "conv_transpose2d"; + break; + } + if (prepack_weights) { + kernel_name += "_prepack_weights"; + } else if (clamp_out) { + kernel_name += "_clamp"; + } + add_dtype_suffix(kernel_name, graph.dtype_of(out)); + + if (prepack_weights) { + add_dtype_suffix(kernel_name, graph.get_staging_dtype_for(weight)); + } + + return VK_KERNEL_FROM_STR(kernel_name); +} + +std::vector get_final_sizes( + const std::vector& original_sizes, + const Conv2dMethod method) { + int64_t batch_padded = utils::align_up_4(utils::val_at(-4, original_sizes)); + int64_t channels_padded = + utils::align_up_4(utils::val_at(-3, original_sizes)); + int64_t height = utils::val_at(-2, original_sizes); + int64_t width = utils::val_at(-1, original_sizes); + + switch (method) { + case Conv2dMethod::Depthwise: + return std::vector{4, batch_padded / 4, height * width}; + case Conv2dMethod::Pointwise: + case Conv2dMethod::SlidingWindow: + return std::vector{ + 4, batch_padded * height / 4, channels_padded * width}; + case Conv2dMethod::Transposed: + return std::vector{ + 4, channels_padded * height / 4, batch_padded * width}; + } +} + +ValueRef prepack_weights( + ComputeGraph& graph, + const ValueRef vref, + const Conv2dMethod method) { + const auto original_sizes = graph.sizes_of(vref); + const auto final_sizes = get_final_sizes(original_sizes, method); + + ValueRef v = graph.add_tensor( + final_sizes, + graph.dtype_of(vref), + utils::kTexture2D, + utils::kChannelsPacked); + + vkapi::ShaderInfo shader = + get_conv2d_shader(graph, v, /*prepack_weights = */ true, method, vref); + + const auto original_sizes_pc = + utils::make_ivec4(original_sizes, /*reverse = */ true); + graph.prepack_nodes().emplace_back(new PrepackNode( + graph, + shader, + graph.create_gwg(v), + graph.create_lwg(v), + vref, + v, + {}, + // Specialization constants + {graph.packed_dim_of(v)}, + {graph.sizes_pc_of(v), + PushConstantDataInfo(&original_sizes_pc, sizeof(original_sizes_pc))})); + + return v; +} + +namespace { + +struct Conv2dParams final { + utils::ivec2 overlay_region; + int in_group_size; +}; + +} // namespace + +struct OutputParams final { + float out_min; + float out_max; +}; + +Conv2dParams create_conv2d_params( + ComputeGraph& graph, + const ValueRef weight, + const Kernel2dParams& p, + const bool transposed) { + const auto& overlay_region = utils::make_ivec2({ + p.kernel_size[0] + (p.kernel_size[0] - 1) * (p.dilation[0] - 1), + p.kernel_size[1] + (p.kernel_size[1] - 1) * (p.dilation[1] - 1), + }); + const auto weight_sizes = graph.sizes_of(weight); + const int32_t in_group_size = utils::safe_downcast( + utils::align_up_4(transposed ? weight_sizes.at(0) : weight_sizes.at(1))); + return {overlay_region, in_group_size}; +} + +void check_conv2d_params(const Kernel2dParams& p, const bool transposed) { + if (transposed) { + if (p.dilation[0] > 1 || p.dilation[1] > 1) { + VK_THROW( + "aten.convolution.default: transposed = true, dilation > 1 is not supported yet!"); + } + } +} + +GlobalWorkGrid create_conv2d_gwg( + ComputeGraph& graph, + const Conv2dMethod method, + const ValueRef out, + const ValueRef weight_data, + const bool stride_equals_dilation) { + (void)weight_data; + (void)stride_equals_dilation; + if (method == Conv2dMethod::Pointwise) { + const utils::uvec3 image_extents = graph.logical_limits_of(out); + return GlobalWorkGrid( + {utils::div_up(image_extents[0u], 1u), + utils::div_up(image_extents[1u], 4u), + image_extents[2u]}, + kTiledWorkGrid); + } else { + return graph.create_gwg(out); + } +} + +// Determines which convolution method a dispatch uses. +// +// Depthwise and transposed convolutions have shader names of their own, but +// the name alone cannot separate pointwise from sliding window: the sliding +// window shader is itself named "conv2d", and a pointwise convolution also +// takes that name when its weights are prepacked. Those two are therefore +// separated by the weight's spatial extent. Shared by the global and local +// workgroup size functions below so that the two cannot disagree about the +// same dispatch. +Conv2dMethod infer_conv2d_method_from_shader( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const ValueRef weight_data) { + const std::string& kernel_name = shader.kernel_name; + // Checked before the plain "conv2d" test below, which "conv2d_dw" and + // "conv2d_pw" would otherwise match too. + if (kernel_name.find("conv2d_dw") != std::string::npos) { + return Conv2dMethod::Depthwise; + } + if (kernel_name.find("conv2d_pw") != std::string::npos) { + return Conv2dMethod::Pointwise; + } + if (kernel_name.find("conv_transpose2d") != std::string::npos) { + return Conv2dMethod::Transposed; + } + const auto& weight_sizes = graph->get_tref(weight_data)->sizes; + if (weight_sizes.at(2) == 1 && weight_sizes.at(3) == 1) { + return Conv2dMethod::Pointwise; + } + return Conv2dMethod::SlidingWindow; +} + +// Custom global workgroup size function for conv2d +GlobalWorkGrid conv2d_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + const ValueRef out = args.at(0).refs.at(0); + const ValueRef weight_data = resize_args.at(0); + + const Conv2dMethod method = + infer_conv2d_method_from_shader(graph, shader, weight_data); + + // Determine stride_equals_dilation from shader name + bool stride_equals_dilation = + shader.kernel_name.find("_sned") == std::string::npos; + + const GlobalWorkGrid wg_size = create_conv2d_gwg( + *graph, method, out, weight_data, stride_equals_dilation); + + if (method == Conv2dMethod::Pointwise) { + utils::uvec3 pointwise_wg_size = {wg_size[0] * wg_size[1], wg_size[2], 1u}; + + if (shader.kernel_name.find("s1p0") != std::string::npos) { + pointwise_wg_size[0] *= 4; + } + return GlobalWorkGrid(pointwise_wg_size, kTiledWorkGrid); + } + + return wg_size; +} + +// Custom local workgroup size function for conv2d +LocalWorkGroup conv2d_lwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args, + const std::vector& resize_args) { + (void)args; + + const ValueRef weight_data = resize_args.at(0); + const Conv2dMethod method = + infer_conv2d_method_from_shader(graph, shader, weight_data); + + if (method == Conv2dMethod::Pointwise) { + uint32_t lwg_y = 1; + if (gwg[1] % 8 == 0) { + lwg_y = 8; + } else if (gwg[1] % 4 == 0) { + lwg_y = 4; + } else if (gwg[1] % 2 == 0) { + lwg_y = 2; + } + return LocalWorkGroup(64u / lwg_y, lwg_y, 1u); + } else { + return graph->create_lwg(gwg); + } +} + +void add_conv2d_direct_node( + ComputeGraph& graph, + const ValueRef in, + const ValueRef arg_weight, + const ValueRef arg_bias, + const ValueRef weight_data, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef transposed, + const ValueRef output_padding, + const ValueRef out, + const vkapi::ShaderInfo& shader, + const Kernel2dParams& kernel_params, + const Conv2dParams& extra_params, + const OutputParams& out_params, + const int64_t groups_val) { + vkapi::ParamsBindList param_buffers = { + graph.logical_limits_ubo(out), + graph.sizes_ubo(in), + graph.create_params_buffer(kernel_params), + graph.create_params_buffer(extra_params), + graph.create_params_buffer(out_params), + }; + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + shader, + conv2d_gwg, + conv2d_lwg, + // Inputs and Outputs + {{out, vkapi::kWrite}, {{in, arg_weight, arg_bias}, vkapi::kRead}}, + // Shader params buffers + param_buffers, + // Push Constants + {}, + // Specialization Constants + {utils::safe_downcast(groups_val)}, + // Resize Args + {weight_data, stride, padding, dilation, transposed, output_padding}, + // Resizing Logic + resize_conv2d_node)); +} + +void conv2d_direct_impl( + ComputeGraph& graph, + const ValueRef in, + const ValueRef weight_data, + const ValueRef bias, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef transposed, + const ValueRef output_padding, + const ValueRef groups, + const ValueRef out, + const bool clamp_out, + const float out_min_val, + const float out_max_val) { + const bool transposed_val = graph.get_bool(transposed); + const int64_t groups_val = graph.get_int(groups); + const Conv2dMethod method = + transposed_val ? Conv2dMethod::Transposed : Conv2dMethod::SlidingWindow; + + const Kernel2dParams kernel_params = create_kernel2d_params( + graph, + weight_data, + /*kernel_size_only = */ false, + stride, + padding, + dilation); + + ValueRef arg_weight = prepack_weights(graph, weight_data, method); + ValueRef arg_bias = prepack_biases( + graph, + bias, + weight_data, + transposed_val, + /* storage_type = */ utils::kTexture2D, + /* memory_layout = */ utils::kWidthPacked); + + const std::vector in_sizes = graph.sizes_of(in); + if (in_sizes.at(0) > 1) { + VK_THROW("conv2d: input batch size > 1 is not supported yet!"); + } + + check_conv_args(graph, in, out); + + Conv2dParams extra_params = + create_conv2d_params(graph, weight_data, kernel_params, transposed_val); + + const bool stride_equals_dilation = + (kernel_params.stride[0] == kernel_params.dilation[0] && + kernel_params.stride[1] == kernel_params.dilation[1]); + + const bool stride_1_padding_0 = + (kernel_params.stride[0] == 1 && kernel_params.stride[1] == 1 && + kernel_params.padding[0] == 0 && kernel_params.padding[1] == 0); + + OutputParams out_params = {out_min_val, out_max_val}; + + check_conv2d_params(kernel_params, transposed_val); + + vkapi::ShaderInfo shader = get_conv2d_shader( + graph, + out, + /*prepack_weights = */ false, + method, + weight_data, + clamp_out, + stride_equals_dilation, + stride_1_padding_0); + + add_conv2d_direct_node( + graph, + in, + arg_weight, + arg_bias, + weight_data, + stride, + padding, + dilation, + transposed, + output_padding, + out, + shader, + kernel_params, + extra_params, + out_params, + groups_val); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.h new file mode 100644 index 00000000000..ef3ea388044 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dDirect.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +// Dispatches the direct conv2d shader, or the transposed conv2d shader when +// `transposed` is true; the two share weight prepacking and dispatch logic. +void conv2d_direct_impl( + ComputeGraph& graph, + const ValueRef in, + const ValueRef weight_data, + const ValueRef bias, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef transposed, + const ValueRef output_padding, + const ValueRef groups, + const ValueRef out, + const bool clamp_out, + const float out_min_val, + const float out_max_val); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv2dPW.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dPW.cpp similarity index 97% rename from backends/vulkan/runtime/graph/ops/impl/Conv2dPW.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dPW.cpp index 3c227448cf2..736bb306f77 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Conv2dPW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/Conv2dPW.cpp @@ -6,10 +6,11 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include +#include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/ConvolutionUtils.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/ConvolutionUtils.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/ConvolutionUtils.h rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/ConvolutionUtils.h diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv2dGemm.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dGemm.cpp similarity index 98% rename from backends/vulkan/runtime/graph/ops/impl/Conv2dGemm.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dGemm.cpp index 6285dc92d4a..55c6582335b 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Conv2dGemm.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dGemm.cpp @@ -6,13 +6,13 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include -#include -#include #include +#include +#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv2dGemm.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dGemm.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Conv2dGemm.h rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dGemm.h diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv2dIm2Col.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dIm2Col.cpp similarity index 98% rename from backends/vulkan/runtime/graph/ops/impl/Conv2dIm2Col.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dIm2Col.cpp index e1bbded799e..e4dcf6592c2 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Conv2dIm2Col.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dIm2Col.cpp @@ -6,7 +6,7 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/Conv2dIm2Col.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dIm2Col.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Conv2dIm2Col.h rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/im2col/Conv2dIm2Col.h diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dRoute.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2d.cpp similarity index 70% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2dRoute.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2d.cpp index 780dfcbfbf3..290207295ca 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dRoute.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2d.cpp @@ -6,9 +6,13 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include -#include +#include + +#include +#include +#include #include #include @@ -144,4 +148,55 @@ bool should_use_q8ta_conv2d_im2col(const Q8taConv2dRouteParams& params) { (params.in_channels_per_group >= 32 || spatial_out <= 4096); } +void q8ta_conv2d(ComputeGraph& graph, const std::vector& args) { + const ValueRef input = args.at(0); + const ValueRef kernel_size_ref = args.at(9); + const ValueRef groups_ref = args.at(13); + const ValueRef output = args.at(15); + + const int64_t groups = graph.extract_scalar(groups_ref); + // Valid models always carry groups >= 1; fail fast on corrupt input + // instead of dividing channel counts by zero downstream (both this + // dispatcher and q8ta_conv2d_general_impl divide by groups). + VK_CHECK_COND(groups > 0, "q8ta_conv2d requires groups >= 1"); + const int64_t in_channels = graph.size_at(-3, input); + const int64_t in_channels_per_group = in_channels / groups; + const int64_t batch = graph.size_at(-4, input); + + const int64_t H_out = graph.size_at(-2, output); + const int64_t W_out = graph.size_at(-1, output); + const int64_t out_channels = graph.size_at(-3, output); + int64_t kernel_height; + int64_t kernel_width; + { + const auto kernel_size = graph.get_int_list(kernel_size_ref); + kernel_height = kernel_size->at(0); + kernel_width = kernel_size->at(1); + } + + const bool use_im2col = should_use_q8ta_conv2d_im2col({ + graph.device_is_mali(), + graph.can_use_int8_dot_product(), + static_cast(graph.max_buffer_numel()), + batch, + groups, + in_channels_per_group, + out_channels, + kernel_height, + kernel_width, + H_out, + W_out, + }); + + if (use_im2col) { + q8ta_conv2d_im2col(graph, args); + } else { + q8ta_conv2d_general_impl(graph, args); + } +} + +REGISTER_OPERATORS { + VK_REGISTER_OP(et_vk.q8ta_conv2d.default, q8ta_conv2d); +} + } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dRoute.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2d.h similarity index 84% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2dRoute.h rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2d.h index 9f60b6fd372..549ab2b25e8 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dRoute.h +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2d.h @@ -8,6 +8,8 @@ #pragma once +#include + #include namespace vkcompute { @@ -30,4 +32,6 @@ struct Q8taConv2dRouteParams final { bool should_use_q8ta_conv2d_im2col(const Q8taConv2dRouteParams& params); +void q8ta_conv2d(ComputeGraph& graph, const std::vector& args); + } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.cpp new file mode 100644 index 00000000000..bde1492ca18 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.cpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +namespace vkcompute { + +ActivationType activation_type_from_string(const std::string& activation) { + if (activation == "none") { + return ActivationType::kNone; + } else if (activation == "relu") { + return ActivationType::kRelu; + } + VK_THROW("Unknown activation type: ", activation); +} + +bool q8ta_conv2d_check_packed_dim_info(const api::PackedDimInfo& info) { + return info.packed_dim == WHCN::kChannelsDim && + info.packed_dim_block_size == 4 && + info.outer_packed_dim == WHCN::kWidthDim && + (info.outer_packed_dim_block_size == 1 || + info.outer_packed_dim_block_size == 4); +} + +bool q8ta_conv2d_check_4w4c_packed_dim_info(const api::PackedDimInfo& info) { + return info.packed_dim == WHCN::kChannelsDim && + info.packed_dim_block_size == 4 && + info.outer_packed_dim == WHCN::kWidthDim && + info.outer_packed_dim_block_size == 4; +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.h new file mode 100644 index 00000000000..5c9cb45d55e --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +#include + +namespace vkcompute { + +enum class ActivationType : uint32_t { + kNone = 0, + kRelu = 1, +}; + +ActivationType activation_type_from_string(const std::string& activation); + +bool q8ta_conv2d_check_packed_dim_info(const api::PackedDimInfo& info); + +bool q8ta_conv2d_check_4w4c_packed_dim_info(const api::PackedDimInfo& info); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dDW.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDW.cpp similarity index 97% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2dDW.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDW.cpp index a504d9ab3f6..b0696bb5442 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dDW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDW.cpp @@ -6,12 +6,14 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include + +#include #include #include -#include #include +#include #include #include @@ -437,7 +439,9 @@ void add_q8ta_conv2d_dw_node( // High level operator impl // -void q8ta_conv2d_dw(ComputeGraph& graph, const std::vector& args) { +void q8ta_conv2d_dw_impl( + ComputeGraph& graph, + const std::vector& args) { int32_t idx = 0; const ValueRef packed_int8_input = args.at(idx++); const ValueRef input_scale = args.at(idx++); @@ -509,7 +513,7 @@ void q8ta_conv2d_dw(ComputeGraph& graph, const std::vector& args) { } REGISTER_OPERATORS { - VK_REGISTER_OP(et_vk.q8ta_conv2d_dw.default, q8ta_conv2d_dw); + VK_REGISTER_OP(et_vk.q8ta_conv2d_dw.default, q8ta_conv2d_dw_impl); } } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDW.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDW.h new file mode 100644 index 00000000000..cbff43acfb9 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDW.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +namespace vkcompute { + +ValueRef prepack_quantized_conv2d_dw_weight( + ComputeGraph& graph, + const QuantizationConfig& weight_quant_config, + const ValueRef weight_data, + const ValueRef kernel_size); + +void add_q8ta_conv2d_dw_node( + ComputeGraph& graph, + const ValueRef packed_int8_input, + const ValueRef input_scale, + const ValueRef input_zp, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef output_scale, + const ValueRef output_zp, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef kernel_size, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef groups, + const uint32_t activation_type, + const ValueRef packed_int8_output); + +void add_conv2d_dw_q8ta_q8csw_q8to_4w4c_node( + ComputeGraph& graph, + const ValueRef packed_int8_input, + const ValueRef input_scale, + const ValueRef input_zp, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef output_scale, + const ValueRef output_zp, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef kernel_size, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef groups, + const ValueRef packed_int8_output); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDirect.cpp similarity index 84% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDirect.cpp index 75f65d7a2a0..d1a3edba2de 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2d.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDirect.cpp @@ -6,43 +6,19 @@ * LICENSE file in the root directory of this source tree. */ -#include -#include +#include #include #include -#include #include +#include +#include #include #include namespace vkcompute { -ActivationType activation_type_from_string(const std::string& activation) { - if (activation == "none") { - return ActivationType::kNone; - } else if (activation == "relu") { - return ActivationType::kRelu; - } - VK_THROW("Unknown activation type: ", activation); -} - -bool q8ta_conv2d_check_packed_dim_info(const api::PackedDimInfo& info) { - return info.packed_dim == WHCN::kChannelsDim && - info.packed_dim_block_size == 4 && - info.outer_packed_dim == WHCN::kWidthDim && - (info.outer_packed_dim_block_size == 1 || - info.outer_packed_dim_block_size == 4); -} - -bool q8ta_conv2d_check_4w4c_packed_dim_info(const api::PackedDimInfo& info) { - return info.packed_dim == WHCN::kChannelsDim && - info.packed_dim_block_size == 4 && - info.outer_packed_dim == WHCN::kWidthDim && - info.outer_packed_dim_block_size == 4; -} - // // Workgroup size selection functions // @@ -160,7 +136,8 @@ ValueRef prepack_quantized_conv2d_weight( VK_CHECK_COND(weight_quant_config.nbits == 8); VK_CHECK_COND(weight_quant_config.is_symmetric); - const int32_t groups_val = graph.get_int(groups); + const int32_t groups_val = + utils::safe_downcast(graph.get_int(groups)); const int64_t OC = graph.size_at(-3, output); const int64_t IC = graph.size_at(-3, input) / groups_val; @@ -397,7 +374,7 @@ void add_q8ta_conv2d_node( // High level operator impl // -void q8ta_conv2d_general( +void q8ta_conv2d_general_impl( ComputeGraph& graph, const std::vector& args) { int32_t idx = 0; @@ -479,56 +456,8 @@ void q8ta_conv2d_general( packed_int8_output); } -void q8ta_conv2d(ComputeGraph& graph, const std::vector& args) { - const ValueRef input = args.at(0); - const ValueRef kernel_size_ref = args.at(9); - const ValueRef groups_ref = args.at(13); - const ValueRef output = args.at(15); - - const int64_t groups = graph.extract_scalar(groups_ref); - // Valid models always carry groups >= 1; fail fast on corrupt input - // instead of dividing channel counts by zero downstream (both this - // dispatcher and q8ta_conv2d_general divide by groups). - VK_CHECK_COND(groups > 0, "q8ta_conv2d requires groups >= 1"); - const int64_t in_channels = graph.size_at(-3, input); - const int64_t in_channels_per_group = in_channels / groups; - const int64_t batch = graph.size_at(-4, input); - - const int64_t H_out = graph.size_at(-2, output); - const int64_t W_out = graph.size_at(-1, output); - const int64_t out_channels = graph.size_at(-3, output); - int64_t kernel_height; - int64_t kernel_width; - { - const auto kernel_size = graph.get_int_list(kernel_size_ref); - kernel_height = kernel_size->at(0); - kernel_width = kernel_size->at(1); - } - - const bool use_im2col = should_use_q8ta_conv2d_im2col({ - graph.device_is_mali(), - graph.can_use_int8_dot_product(), - static_cast(graph.max_buffer_numel()), - batch, - groups, - in_channels_per_group, - out_channels, - kernel_height, - kernel_width, - H_out, - W_out, - }); - - if (use_im2col) { - q8ta_conv2d_im2col(graph, args); - } else { - q8ta_conv2d_general(graph, args); - } -} - REGISTER_OPERATORS { - VK_REGISTER_OP(et_vk.q8ta_conv2d.default, q8ta_conv2d); - VK_REGISTER_OP(et_vk.q8ta_conv2d_general.default, q8ta_conv2d_general); + VK_REGISTER_OP(et_vk.q8ta_conv2d_general.default, q8ta_conv2d_general_impl); } } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDirect.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDirect.h new file mode 100644 index 00000000000..e1960303e1f --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dDirect.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +namespace vkcompute { + +ValueRef prepack_quantized_conv2d_weight( + ComputeGraph& graph, + const QuantizationConfig& weight_quant_config, + const ValueRef weight_data, + const ValueRef input, + const ValueRef output, + const ValueRef groups, + const ValueRef kernel_size); + +void add_q8ta_conv2d_node( + ComputeGraph& graph, + const ValueRef packed_int8_input, + const ValueRef packed_int8_input_im2col, + const ValueRef input_scale, + const ValueRef input_zp, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef output_scale, + const ValueRef output_zp, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef kernel_size, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef groups, + const uint32_t activation_type, + const ValueRef packed_int8_output); + +void q8ta_conv2d_general_impl( + ComputeGraph& graph, + const std::vector& args); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dPW.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dPW.cpp similarity index 98% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2dPW.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dPW.cpp index 454a115498b..d4bb4dbab38 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dPW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dPW.cpp @@ -6,10 +6,13 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include + +#include #include #include +#include #include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dPW.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dPW.h new file mode 100644 index 00000000000..97878ee122c --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dPW.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +void add_q8ta_conv2d_pw_node( + ComputeGraph& graph, + const bool use_unsigned_dot, + const ValueRef packed_int8_input, + const ValueRef input_scale, + const ValueRef input_zp, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef output_scale, + const ValueRef output_zp, + const ValueRef bias_data, + const ValueRef packed_bias, + const uint32_t activation_type, + const ValueRef packed_int8_output, + const int32_t groups = 1, + const ValueRef conv_input = kDummyValueRef, + const ValueRef kernel_size = kDummyValueRef, + const ValueRef stride = kDummyValueRef, + const ValueRef padding = kDummyValueRef, + const ValueRef dilation = kDummyValueRef, + const bool is_im2col = false, + const ValueRef stream_row_offset_ref = kDummyValueRef, + const ValueRef max_im2col_rows_ref = kDummyValueRef); + +constexpr int64_t kMaxUnsignedDotAccumulatorBytes = 33025; + +bool can_use_unsigned_pw_dot( + const vkapi::Adapter& adapter, + int64_t k_per_group); + +void q8ta_conv2d_pw_impl( + ComputeGraph& graph, + bool use_unsigned_dot, + const std::vector& args); + +void q8ta_conv2d_pw(ComputeGraph& graph, const std::vector& args); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dIm2Col.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.cpp similarity index 96% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2dIm2Col.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.cpp index 062e5ea1d8d..0fcb2105b2e 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dIm2Col.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.cpp @@ -6,15 +6,18 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include + +#include +#include +#include #include #include -#include -#include #include +#include +#include #include -#include #include #include @@ -372,7 +375,7 @@ void q8ta_conv2d_im2col_impl( full_im2col_sizes.at(3), graph.max_buffer_numel()); if (!stream_plan.feasible) { - q8ta_conv2d_general(graph, args); + q8ta_conv2d_general_impl(graph, args); return; } VK_CHECK_COND( diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.h new file mode 100644 index 00000000000..c1520726111 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/im2col/Q8taConv2dIm2Col.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +inline constexpr int64_t kQ8taConv2dIm2ColScratchBudgetBytes = 16 * 1024 * 1024; +inline constexpr int64_t kQ8taConv2dMaxRowsPerTile = 65535; + +struct Q8taConv2dStreamPlan final { + int64_t aligned_out_width; + int64_t rows_per_tile; + int64_t num_tiles; + int64_t scratch_bytes; + bool feasible; +}; + +Q8taConv2dStreamPlan make_q8ta_conv2d_stream_plan( + int64_t batch, + int64_t flattened_kernel_size, + int64_t out_height, + int64_t out_width, + int64_t scratch_budget_bytes); + +// max_buffer_bytes is Adapter::max_buffer_numel(), which returns +// maxStorageBufferRange in bytes (not elements) — directly comparable with +// the byte-denominated scratch budget. +Q8taConv2dStreamPlan make_q8ta_conv2d_stream_plan_for_device( + int64_t batch, + int64_t flattened_kernel_size, + int64_t out_height, + int64_t out_width, + uint64_t max_buffer_bytes); + +std::vector calculate_q8ta_im2col_sizes( + ComputeGraph* graph, + const ValueRef& input, + const ValueRef& output, + const ValueRef& kernel_size, + const ValueRef& groups); + +void add_q8ta_im2col_node( + ComputeGraph& graph, + const ValueRef packed_int8_input, + const ValueRef kernel_size, + const ValueRef stride, + const ValueRef padding, + const ValueRef dilation, + const ValueRef groups, + const ValueRef packed_int8_output, + const ValueRef packed_int8_im2col, + const int32_t zp, + const ValueRef stream_row_offset_ref, + const ValueRef max_im2col_rows_ref = kDummyValueRef); + +void q8ta_conv2d_im2col(ComputeGraph& graph, const std::vector& args); + +void q8ta_conv2d_im2col_impl( + ComputeGraph& graph, + bool use_unsigned_dot, + const std::vector& args); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dTransposed.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/transposed/Q8taConv2dTransposed.cpp similarity index 95% rename from backends/vulkan/runtime/graph/ops/impl/Q8taConv2dTransposed.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/transposed/Q8taConv2dTransposed.cpp index 91d745fb1fa..ff5cac10a0b 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taConv2dTransposed.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/transposed/Q8taConv2dTransposed.cpp @@ -6,13 +6,16 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include + +#include +#include #include #include -#include #include +#include #include #include @@ -228,7 +231,7 @@ void add_q8ta_conv2d_transposed_node( resize_q8ta_conv2d_transposed_node)); } -void q8ta_conv2d_transposed( +void q8ta_conv2d_transposed_impl( ComputeGraph& graph, const std::vector& args) { int32_t idx = 0; @@ -313,7 +316,8 @@ void q8ta_conv2d_transposed( } REGISTER_OPERATORS { - VK_REGISTER_OP(et_vk.q8ta_conv2d_transposed.default, q8ta_conv2d_transposed); + VK_REGISTER_OP( + et_vk.q8ta_conv2d_transposed.default, q8ta_conv2d_transposed_impl); } } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/transposed/Q8taConv2dTransposed.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/transposed/Q8taConv2dTransposed.h new file mode 100644 index 00000000000..92607284b00 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/transposed/Q8taConv2dTransposed.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +void q8ta_conv2d_transposed_impl( + ComputeGraph& graph, + const std::vector& args); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedConvolution.cpp b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.cpp similarity index 98% rename from backends/vulkan/runtime/graph/ops/impl/QuantizedConvolution.cpp rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.cpp index 30415cd4f08..d58f669362f 100644 --- a/backends/vulkan/runtime/graph/ops/impl/QuantizedConvolution.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.cpp @@ -9,13 +9,15 @@ #include #include -#include -#include #include #include -#include -#include #include +#include +#include +#include +#include +#include +#include #include namespace vkcompute { @@ -1341,7 +1343,7 @@ void conv2d_q8ta_q8csw_q8to( // Test operators // -void test_conv2d_q8ta_q8csw_q8to( +void test_conv2d_q8ta_q8csw_q8to_impl( ComputeGraph& graph, const std::vector& args) { int32_t idx = 0; @@ -1407,7 +1409,8 @@ REGISTER_OPERATORS { VK_REGISTER_OP(et_vk.conv2d_q8ta_q8csw.default, conv2d_q8ta_q8csw); VK_REGISTER_OP(et_vk.conv2d_q8csw.default, conv2d_q8csw); VK_REGISTER_OP( - etvk.test_conv2d_q8ta_q8csw_q8to.default, test_conv2d_q8ta_q8csw_q8to); + etvk.test_conv2d_q8ta_q8csw_q8to.default, + test_conv2d_q8ta_q8csw_q8to_impl); VK_REGISTER_OP(et_vk.conv2d_q8ta_q8csw_q8to.default, conv2d_q8ta_q8csw_q8to); VK_REGISTER_OP( et_vk.conv2d_q8ta_q8csw_q8to_dw.default, conv2d_q8ta_q8csw_q8to); diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedConvolution.h b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.h similarity index 78% rename from backends/vulkan/runtime/graph/ops/impl/QuantizedConvolution.h rename to backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.h index 8913e639f0e..6bc453d0d27 100644 --- a/backends/vulkan/runtime/graph/ops/impl/QuantizedConvolution.h +++ b/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/quantized_legacy/QuantizedConvolution.h @@ -8,7 +8,7 @@ #pragma once -#include +#include namespace vkcompute { diff --git a/backends/vulkan/runtime/graph/ops/impl/GemmCommon.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/GemmCommon.cpp similarity index 98% rename from backends/vulkan/runtime/graph/ops/impl/GemmCommon.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/GemmCommon.cpp index b763dcaeb38..61e053e0751 100644 --- a/backends/vulkan/runtime/graph/ops/impl/GemmCommon.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/GemmCommon.cpp @@ -6,7 +6,7 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/GemmCoopmat.h b/backends/vulkan/runtime/graph/ops/impl/gemm/GemmCommon.h similarity index 59% rename from backends/vulkan/runtime/graph/ops/impl/GemmCoopmat.h rename to backends/vulkan/runtime/graph/ops/impl/gemm/GemmCommon.h index 7be7e8bc157..eb1c1c74c24 100644 --- a/backends/vulkan/runtime/graph/ops/impl/GemmCoopmat.h +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/GemmCommon.h @@ -9,13 +9,40 @@ #pragma once #include +#include namespace vkcompute { -// Cooperative-matrix GEMM dispatch. The coopmat shader (coopmat_mm.glsl) is -// shared across linear/matmul; the two add_*_node entry points below differ -// only in input layout (prepacked weight vs runtime mat2) and bias handling. -// +// Helpers shared by the linear/matmul tiled and coopmat dispatch paths. + +// Prepack a floating-point weight tensor for the linear/matmul kernels. +// Source layout: [N, K] (is_transposed=true) or [K, N] (is_transposed=false), +// optionally batched. Output layout: 4OC x 4IC blocked, packed as +// kWidthPacked. When force_buffer is true, the packed tensor uses buffer +// storage (required by the coopmat shader); otherwise texture2d is used +// when it fits within max_texture2d_dim. +ValueRef prepack_fp_linear_weight( + ComputeGraph& graph, + const ValueRef weight_data, + bool is_transposed, + int64_t B, + bool force_buffer = false); + +// Resize logic for linear-shaped output: takes M from the input's penultimate +// dim, N from resize_args[0], optional batch from input's leading dim. +void resize_linear_node( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args); + +// Resize logic for matmul (mat1 @ mat2): output rows from mat1, cols from +// mat2, batch dims propagated from mat1. +void resize_matmul_tiled_node( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args); + +// Cooperative-matrix GEMM (coopmat_mm.glsl, shared by linear and matmul). // Tile dimensions match the coopmat_mm.yaml defaults; they participate in // the eligibility check below because the shader has no partial-tile or // K-tail handling — misaligned shapes must fall back to the tiled path. @@ -52,19 +79,4 @@ inline bool is_coopmat_eligible( N % kCoopmatTileN == 0 && K % kCoopmatTileK == 0; } -void add_linear_coopmat_node( - ComputeGraph& graph, - const ValueRef input, - const ValueRef packed_weight, - const ValueRef packed_bias, - bool has_bias, - const ValueRef out, - int32_t weight_B = 1); - -void add_matmul_coopmat_node( - ComputeGraph& graph, - const ValueRef mat1, - const ValueRef mat2, - const ValueRef out); - } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Linear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/Linear.cpp similarity index 95% rename from backends/vulkan/runtime/graph/ops/impl/Linear.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/Linear.cpp index 86f27b63586..722cde98515 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Linear.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/Linear.cpp @@ -9,11 +9,11 @@ #include #include -#include -#include -#include -#include #include +#include +#include +#include +#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/Linear.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/Linear.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Linear.h rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/Linear.h diff --git a/backends/vulkan/runtime/graph/ops/impl/GemmCoopmat.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/LinearCoopmat.cpp similarity index 53% rename from backends/vulkan/runtime/graph/ops/impl/GemmCoopmat.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/LinearCoopmat.cpp index 6b62fa6a9ba..5b3e898cff6 100644 --- a/backends/vulkan/runtime/graph/ops/impl/GemmCoopmat.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/LinearCoopmat.cpp @@ -6,19 +6,15 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include -#include +#include -#include -#include #include namespace vkcompute { -// ── Linear coopmat ── - static vkapi::ShaderInfo pick_linear_coopmat_shader( ComputeGraph* graph, const std::vector& args, @@ -40,8 +36,10 @@ static GlobalWorkGrid pick_linear_coopmat_gwg( (void)resize_args; const ValueRef out = args.at(0).refs.at(0); const auto out_sizes = graph->sizes_of(out); - uint32_t M = out_sizes.at(out_sizes.size() - 2); - uint32_t N = out_sizes.at(out_sizes.size() - 1); + uint32_t M = + utils::safe_downcast(out_sizes.at(out_sizes.size() - 2)); + uint32_t N = + utils::safe_downcast(out_sizes.at(out_sizes.size() - 1)); uint32_t num_tiles_n = utils::div_up(N, kCoopmatTileN); uint32_t num_tiles_m = utils::div_up(M, kCoopmatTileM); // Each workgroup processes one WG_TILE_M x WG_TILE_N output tile via @@ -81,7 +79,8 @@ void add_linear_coopmat_node( "linear_coopmat requires buffer storage"); std::vector out_sizes = graph.sizes_of(out); - int32_t orig_N = utils::safe_downcast(out_sizes.back()); + int32_t orig_N = + utils::safe_downcast(out_sizes.at(out_sizes.size() - 1)); ValueRef orig_N_ref = graph.add_scalar(static_cast(orig_N)); ValueRef has_bias_ref = graph.add_scalar(has_bias); @@ -109,79 +108,4 @@ void add_linear_coopmat_node( resize_linear_node)); } -// ── Matmul coopmat ── - -static vkapi::ShaderInfo pick_matmul_coopmat_shader( - ComputeGraph* graph, - const std::vector& args, - const std::vector& resize_args) { - (void)resize_args; - const ValueRef out = args.at(0).refs.at(0); - std::string kernel_name = "matmul_coopmat"; - kernel_name.reserve(kShaderNameReserve); - add_dtype_suffix(kernel_name, graph->dtype_of(out)); - return VK_KERNEL_FROM_STR(kernel_name); -} - -static GlobalWorkGrid pick_matmul_coopmat_gwg( - ComputeGraph* graph, - const vkapi::ShaderInfo& shader, - const std::vector& args, - const std::vector& resize_args) { - (void)shader; - (void)resize_args; - const ValueRef out = args.at(0).refs.at(0); - const auto out_sizes = graph->sizes_of(out); - uint32_t M = out_sizes.at(out_sizes.size() - 2); - uint32_t N = out_sizes.at(out_sizes.size() - 1); - uint32_t num_tiles_n = utils::div_up(N, kCoopmatTileN); - uint32_t num_tiles_m = utils::div_up(M, kCoopmatTileM); - // Each workgroup processes one WG_TILE_M x WG_TILE_N output tile via - // cooperative-matrix MMAs across its 4 subgroups. We want the dispatch - // to launch exactly num_tiles_n x num_tiles_m workgroups. - // - // The framework computes the group count as - // group_count = div_up(gwg, lwg) - // (see Context.cpp + Command.cpp). With lwg = (kCoopmatInvocations, - // 1, 1), multiplying num_tiles_n by kCoopmatInvocations cancels the - // div, yielding group_count.x = num_tiles_n. - return GlobalWorkGrid( - {num_tiles_n * kCoopmatInvocations, num_tiles_m, 1u}, - kTiledWorkGrid, - LocalWorkGroup(kCoopmatInvocations, 1u, 1u)); -} - -void add_matmul_coopmat_node( - ComputeGraph& graph, - const ValueRef mat1, - const ValueRef mat2, - const ValueRef out) { - VK_CHECK_COND(graph.packed_dim_of(mat1) == WHCN::kWidthDim); - VK_CHECK_COND(graph.packed_dim_of(mat2) == WHCN::kWidthDim); - VK_CHECK_COND(graph.packed_dim_of(out) == WHCN::kWidthDim); - VK_CHECK_COND( - graph.storage_type_of(out) == utils::kBuffer, - "matmul_coopmat requires buffer storage"); - - ValueRef has_bias_ref = graph.add_scalar(false); - - graph.execute_nodes().emplace_back(new DynamicDispatchNode( - graph, - pick_matmul_coopmat_shader, - pick_matmul_coopmat_gwg, - pick_required_lwg, - // Inputs and Outputs — same binding order as matmul_vec - {{out, vkapi::kWrite}, {{mat1, mat2}, vkapi::kRead}}, - // Shader params buffers — same UBOs as matmul_vec - {graph.sizes_ubo(mat1), graph.sizes_ubo(mat2)}, - // Push Constants - {}, - // Specialization Constants - {}, - // Resize Args - {has_bias_ref}, - // Resizing Logic - resize_matmul_tiled_node)); -} - } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/LinearCoopmat.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/LinearCoopmat.h new file mode 100644 index 00000000000..e5aac00c04e --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/LinearCoopmat.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +void add_linear_coopmat_node( + ComputeGraph& graph, + const ValueRef input, + const ValueRef packed_weight, + const ValueRef packed_bias, + bool has_bias, + const ValueRef out, + int32_t weight_B = 1); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/LinearDw.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/backward/LinearDw.cpp similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/LinearDw.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/backward/LinearDw.cpp diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taLinear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinear.cpp similarity index 96% rename from backends/vulkan/runtime/graph/ops/impl/Q8taLinear.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinear.cpp index 683b7b5ce07..fba34214dab 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taLinear.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinear.cpp @@ -6,14 +6,14 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include -#include -#include #include +#include +#include #include namespace vkcompute { diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taLinear.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinear.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Q8taLinear.h rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinear.h diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinearGemv.cpp similarity index 95% rename from backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinearGemv.cpp index 7a63d33d546..2861bb1d7b9 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinearGemv.cpp @@ -6,14 +6,14 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include #include -#include -#include #include +#include +#include #include namespace vkcompute { diff --git a/backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinearGemv.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Q8taLinearGemv.h rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/q8ta/Q8taLinearGemv.h diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/QuantizedLinear.cpp similarity index 63% rename from backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/QuantizedLinear.cpp index 3f1ea14f20b..3b3e11c8498 100644 --- a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/QuantizedLinear.cpp @@ -6,13 +6,13 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include -#include #include -#include #include +#include +#include #include namespace vkcompute { @@ -51,26 +51,6 @@ void resize_linear_qw_node( graph->virtual_resize(output, new_out_sizes); } -// Per-shader coopmat tile geometry (must match each shader's yaml). -// Workgroup size (wg_size) = SG_GRID_X * SG_GRID_Y * SUBGROUP_SIZE. -// linear_q4gsw_coopmat 128x64x16, 2x2 subgroups x 32 (forced) -> 128 -// linear_dq8ca_q4gsw_coopmat 128x64x32, 2x2 subgroups x 64 -> 256 -// (The int8-MMA shaders stay on wave64: int8 WMMA at forced subgroup 32 -// crashes the Xclipse PAL compiler.) -struct CoopmatTileDims { - uint32_t m; - uint32_t n; - uint32_t k; - // Threads per workgroup = SG_GRID_X * SG_GRID_Y * SUBGROUP_SIZE. MUST match - // the WG_SIZE the shader yaml resolves to, or the launched thread count won't - // match the shader's staging passes (out-of-bounds). - uint32_t wg_size; -}; -// linear_qw_coopmat.yaml: 128x64, 2x2 subgroup grid, sg32 -> WG_SIZE 128. -constexpr CoopmatTileDims kQ4gswCoopmatDims = {128, 64, 16, 128}; -// linear_dq8ca_qw_coopmat.yaml: 128x64, 2x2 grid, sg64 -> WG_SIZE 256. -constexpr CoopmatTileDims kDq8caQ4gswCoopmatDims = {128, 64, 32, 256}; - static CoopmatTileDims coopmat_tile_dims(const std::string& kernel_name) { // Exact prefix matches (the "linear_dq8ca_*" names must not match the // weight-only entries). @@ -88,16 +68,19 @@ GlobalWorkGrid quantized_linear_gwg( const vkapi::ShaderInfo& shader, const std::vector& args, const std::vector& resize_args) { + (void)resize_args; const ValueRef out = args.at(0).refs.at(0); std::vector out_sizes = graph->sizes_of(out); // width - const uint32_t N = utils::val_at(-1, out_sizes); + const uint32_t N = + utils::safe_downcast(utils::val_at(-1, out_sizes)); // height - const uint32_t M = utils::val_at(-2, out_sizes); + const uint32_t M = + utils::safe_downcast(utils::val_at(-2, out_sizes)); // Coopmat variants dispatch a 256-thread WG per 64x64 output tile. Mirrors - // GemmCoopmat.cpp's pick_linear_coopmat_gwg — the multiplication + // LinearCoopmat.cpp's pick_linear_coopmat_gwg — the multiplication // by kCoopmatInvocations cancels the framework's div_up, since // lwg = {256, 1, 1}. if (shader.kernel_name.find("_coopmat") != std::string::npos) { @@ -159,15 +142,15 @@ LocalWorkGroup quantized_linear_lwg( // (M, N, K, dtype, output_storage, group_size) tuple. Preconditions match what // linear_q4gsw_coopmat.glsl assumes; the subgroup_size == 64 check scopes this // to wave64 devices (e.g. AMD RDNA), which the coopmat tiling is tuned for. -static bool can_use_q4gsw_coopmat( +bool can_use_q4gsw_coopmat( ComputeGraph* graph, const ValueRef output, const ValueRef fp_input, int64_t group_size, const ValueRef bias, - int64_t tile_m = kCoopmatTileM, - int64_t tile_n = kCoopmatTileN, - int64_t tile_k = kCoopmatTileK) { + int64_t tile_m, + int64_t tile_n, + int64_t tile_k) { // The coopmat shaders only build HAS_BIAS=false variants, so they would // silently drop a bias. Fall back to the tiled path (which applies bias at // runtime via the apply_bias spec constant) whenever a bias is present. @@ -288,56 +271,6 @@ vkapi::ShaderInfo pick_linear_qw_shader( return VK_KERNEL_FROM_STR(kernel_name); } -vkapi::ShaderInfo pick_linear_dqa_qw_shader( - ComputeGraph* graph, - const std::vector& args, - const std::vector& resize_args) { - (void)resize_args; - - const ValueRef out = args.at(0).refs.at(0); - const ValueRef fp_input = args.at(1).refs.at(0); - const ValueRef int_input = args.at(1).refs.at(1); - (void)int_input; - const ValueRef input_zp = args.at(1).refs.at(4); - const ValueRef int_weight = args.at(1).refs.at(5); - - const bool weight_is_4bit = resize_args.at(0) != kDummyValueRef; - const bool is_gemv_case = is_gemv(graph, fp_input); - - // Use the coopmat shader for 4-bit dq8ca dispatches when the device - // enumerates VK_COMPONENT_TYPE_SINT8_KHR in its cooperative matrix property - // list and the shape aligns; tiled otherwise. - if (weight_is_4bit && !is_gemv_case && - graph->context()->adapter_ptr()->supports_int8_cooperative_matrix()) { - const int64_t group_size = - graph->extract_scalar(resize_args.at(0)); - if (can_use_q4gsw_coopmat( - graph, - out, - fp_input, - group_size, - resize_args.at(2), - kDq8caQ4gswCoopmatDims.m, - kDq8caQ4gswCoopmatDims.n, - kDq8caQ4gswCoopmatDims.k)) { - std::string kernel_name = "linear_dq8ca_q4gsw_coopmat"; - add_storage_type_suffix(kernel_name, graph->storage_type_of(out)); - add_storage_type_suffix(kernel_name, graph->storage_type_of(int_weight)); - add_dtype_suffix(kernel_name, graph->dtype_of(out)); - return VK_KERNEL_FROM_STR(kernel_name); - } - } - - std::string kernel_name = "linear_dq8ca_q4gsw"; - kernel_name += is_gemv_case ? "_coop" : "_tiled"; - add_storage_type_suffix(kernel_name, graph->storage_type_of(out)); - add_storage_type_suffix(kernel_name, graph->storage_type_of(int_weight)); - add_dtype_suffix(kernel_name, graph->dtype_of(out)); - add_zp_dtype_mode_suffix(kernel_name, graph->dtype_of(input_zp)); - - return VK_KERNEL_FROM_STR(kernel_name); -} - // // Prepacking nodes // @@ -500,6 +433,7 @@ void add_linear_qw_node( const ValueRef bias_data, const ValueRef packed_bias, const ValueRef output) { + (void)packed_weight_zeros; // Only certain quantization types supported at the moment VK_CHECK_COND( weight_quant_config.granularity == kPerChannel || @@ -556,160 +490,6 @@ void add_linear_qw_node( resize_linear_qw_node)); } -void add_linear_qa_qw_node( - ComputeGraph& graph, - const QuantizationConfig& input_quant_config, - const QuantizationConfig& weight_quant_config, - const ValueRef fp_input, - const ValueRef packed_int_input, - const ValueRef packed_input_scale, - const ValueRef packed_input_zp, - const ValueRef input_scale_data, - const ValueRef input_zp_data, - const ValueRef weight_data, - const ValueRef packed_weight, - const ValueRef packed_weight_sums, - const ValueRef packed_weight_scales, - const ValueRef group_size, - const ValueRef bias_data, - const ValueRef packed_bias, - const ValueRef output) { - VK_CHECK_COND(input_quant_config.granularity == kPerTensor); - VK_CHECK_COND(input_quant_config.nbits == 8); - VK_CHECK_COND(weight_quant_config.granularity == kPerChannel); - VK_CHECK_COND(weight_quant_config.is_symmetric); - VK_CHECK_COND(weight_quant_config.nbits == 8); - - float scale = graph.extract_scalar(input_scale_data); - int32_t zp = graph.extract_scalar(input_zp_data); - - // Get shader for quantized linear - std::string kernel_name = "linear_q8ta_q8csw_tiled"; - add_storage_type_suffix(kernel_name, graph.storage_type_of(output)); - add_storage_type_suffix(kernel_name, graph.storage_type_of(packed_int_input)); - add_storage_type_suffix(kernel_name, graph.storage_type_of(packed_weight)); - add_dtype_suffix(kernel_name, graph.dtype_of(output)); - vkapi::ShaderInfo shader = VK_KERNEL_FROM_STR(kernel_name); - - vkapi::ParamsBindList param_buffers = { - graph.sizes_ubo(output), graph.sizes_ubo(packed_int_input)}; - - std::vector push_constants = { - PushConstantDataInfo(&scale, sizeof(scale)), - PushConstantDataInfo(&zp, sizeof(zp)), - }; - - uint32_t apply_bias = 1; - if (graph.val_is_none(bias_data)) { - apply_bias = 0; - } - - graph.execute_nodes().emplace_back(new DynamicDispatchNode( - graph, - VK_KERNEL_FROM_STR(kernel_name), - quantized_linear_gwg, - quantized_linear_lwg, - // Inputs and Outputs - {{output, vkapi::kWrite}, - {{packed_int_input, - packed_weight, - packed_weight_sums, - packed_weight_scales, - packed_bias}, - vkapi::kRead}}, - // Shader params buffers - param_buffers, - // Push Constants - push_constants, - // Specialization Constants - {apply_bias}, - // Resize args - {fp_input}, - // Resizing Logic - nullptr)); -} - -void add_linear_dqa_qw_node( - ComputeGraph& graph, - const QuantizationConfig& input_quant_config, - const QuantizationConfig& weight_quant_config, - const ValueRef fp_input, - const ValueRef packed_int_input, - const ValueRef int_input_sums, - const ValueRef packed_input_scale, - const ValueRef packed_input_zp, - const ValueRef input_scale_data, - const ValueRef input_zp_data, - const ValueRef weight_data, - const ValueRef packed_weight, - const ValueRef packed_weight_sums, - const ValueRef packed_weight_scales, - const ValueRef group_size, - const ValueRef bias_data, - const ValueRef packed_bias, - const ValueRef output) { - VK_CHECK_COND(input_quant_config.granularity == kPerChannel); - VK_CHECK_COND(input_quant_config.nbits == 8); - VK_CHECK_COND(input_quant_config.is_dynamic); - - VK_CHECK_COND(weight_quant_config.granularity == kPerGroup); - VK_CHECK_COND(weight_quant_config.is_symmetric); - VK_CHECK_COND(weight_quant_config.nbits == 4); - - vkapi::ParamsBindList param_buffers = { - graph.sizes_ubo(output), graph.sizes_ubo(fp_input)}; - - uint32_t apply_bias = 1; - if (graph.val_is_none(bias_data)) { - apply_bias = 0; - } - - int32_t K4_per_group = 0; - int32_t coopmat_k_iters = 0; - const int32_t K_dim = graph.size_at(-1, fp_input); - if (weight_quant_config.nbits == 4) { - int32_t group_size_val = graph.extract_scalar(group_size); - K4_per_group = utils::div_up(group_size_val, int32_t(4)); - coopmat_k_iters = K_dim / group_size_val; - } - - const ValueRef is_4bit_flag = - weight_quant_config.nbits == 4 ? group_size : kDummyValueRef; - - graph.execute_nodes().emplace_back(new DynamicDispatchNode( - graph, - pick_linear_dqa_qw_shader, - quantized_linear_gwg, - quantized_linear_lwg, - // Inputs and Outputs - {{output, vkapi::kWrite}, - {{fp_input, - packed_int_input, - int_input_sums, - packed_input_scale, - packed_input_zp, - packed_weight, - packed_weight_sums, - packed_weight_scales, - packed_bias}, - vkapi::kRead}}, - // Shader params buffers - param_buffers, - // Push Constants - {}, - // Specialization Constants - // 4th spec const: output width N for coopMatStore (see - // add_linear_qw_node). - {apply_bias, - K4_per_group, - coopmat_k_iters, - graph.size_at(-1, output)}, - // Resize args (resize_args.at(2) = bias_data, read by the coopmat gate) - {is_4bit_flag, weight_data, bias_data}, - // Resizing Logic - resize_linear_qw_node)); -} - // // High level operator impl // @@ -728,6 +508,7 @@ void quantized_linear_impl( const ValueRef group_size, const ValueRef bias_data, const ValueRef output) { + (void)weight_zeros_data; std::vector input_sizes = graph.sizes_of(fp_input); std::vector weight_sizes = graph.sizes_of(weight_data); @@ -898,141 +679,4 @@ void quantized_linear_impl( output); } -void linear_q8ta_q8csw(ComputeGraph& graph, const std::vector& args) { - int32_t idx = 0; - const ValueRef fp_input = args.at(idx++); - const ValueRef input_scale = args.at(idx++); - const ValueRef input_zp = args.at(idx++); - const ValueRef weight_data = args.at(idx++); - const ValueRef weight_sums_data = args.at(idx++); - const ValueRef weight_scales_data = args.at(idx++); - const ValueRef bias_data = args.at(idx++); - const ValueRef output = args.at(idx++); - - const int64_t K = graph.size_at(-1, fp_input); - - QuantizationConfig input_quant_config(8, kPerTensor, {}, false); - QuantizationConfig weight_quant_config(8, kPerChannel, {K}); - - quantized_linear_impl( - graph, - input_quant_config, - weight_quant_config, - fp_input, - input_scale, - input_zp, - weight_data, - weight_sums_data, - weight_scales_data, - kDummyValueRef, // weight_zeros_data - kDummyValueRef, // group_size - bias_data, - output); -} - -void linear_q8csw(ComputeGraph& graph, const std::vector& args) { - int32_t idx = 0; - const ValueRef fp_input = args.at(idx++); - const ValueRef weight_data = args.at(idx++); - const ValueRef weight_scales_data = args.at(idx++); - const ValueRef bias_data = args.at(idx++); - const ValueRef output = args.at(idx++); - - const int64_t K = graph.size_at(-1, fp_input); - - QuantizationConfig input_quant_config(32, kNoQuantization, {}); - QuantizationConfig weight_quant_config(8, kPerChannel, {K}); - - quantized_linear_impl( - graph, - input_quant_config, - weight_quant_config, - fp_input, - kDummyValueRef, // input scale - kDummyValueRef, // input zp - weight_data, - kDummyValueRef, // weight sums - weight_scales_data, - kDummyValueRef, // weight zeros - kDummyValueRef, // group size - bias_data, - output); -} - -// aten._weight_int8pack_mm is what the AOT weight-only int8 fusion -// (FuseQuantizedOpsTransform) emits. It carries the same operands as -// et_vk.linear_q8csw minus the bias, so it runs through the same -// implementation. -void weight_int8pack_mm( - ComputeGraph& graph, - const std::vector& args) { - int32_t idx = 0; - const ValueRef fp_input = args.at(idx++); - const ValueRef weight_data = args.at(idx++); - const ValueRef weight_scales_data = args.at(idx++); - const ValueRef output = args.at(idx++); - - const int64_t K = graph.size_at(-1, fp_input); - - QuantizationConfig input_quant_config(32, kNoQuantization, {}); - QuantizationConfig weight_quant_config(8, kPerChannel, {K}); - - quantized_linear_impl( - graph, - input_quant_config, - weight_quant_config, - fp_input, - kDummyValueRef, // input scale - kDummyValueRef, // input zp - weight_data, - kDummyValueRef, // weight sums - weight_scales_data, - kDummyValueRef, // weight zeros - kDummyValueRef, // group size - kDummyValueRef, // bias - output); -} - -void linear_dq8ca_q4gsw( - ComputeGraph& graph, - const std::vector& args) { - int32_t idx = 0; - const ValueRef fp_input = args.at(idx++); - const ValueRef input_scale = args.at(idx++); - const ValueRef input_zp = args.at(idx++); - const ValueRef weight_data = args.at(idx++); - const ValueRef weight_sums_data = args.at(idx++); - const ValueRef weight_scales_data = args.at(idx++); - const ValueRef group_size = args.at(idx++); - const ValueRef bias_data = args.at(idx++); - const ValueRef output = args.at(idx++); - - const int64_t group_size_val = graph.extract_scalar(group_size); - - QuantizationConfig input_quant_config(8, kPerChannel, {}, false, true); - QuantizationConfig weight_quant_config(4, kPerGroup, {group_size_val}); - - quantized_linear_impl( - graph, - input_quant_config, - weight_quant_config, - fp_input, - input_scale, - input_zp, - weight_data, - weight_sums_data, - weight_scales_data, - kDummyValueRef, // weight_zeros_data - group_size, // group_size - bias_data, - output); -} - -REGISTER_OPERATORS { - VK_REGISTER_OP(et_vk.linear_q8ta_q8csw.default, linear_q8ta_q8csw); - VK_REGISTER_OP(et_vk.linear_q8csw.default, linear_q8csw); - VK_REGISTER_OP(aten._weight_int8pack_mm.default, weight_int8pack_mm); - VK_REGISTER_OP(et_vk.linear_dq8ca_q4gsw.default, linear_dq8ca_q4gsw); -} - } // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/QuantizedLinear.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/QuantizedLinear.h new file mode 100644 index 00000000000..23f70416754 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/QuantizedLinear.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include +#include +#include + +namespace vkcompute { + +// Per-shader coopmat tile geometry (must match each shader's yaml). +// Workgroup size (wg_size) = SG_GRID_X * SG_GRID_Y * SUBGROUP_SIZE. +// linear_q4gsw_coopmat 128x64x16, 2x2 subgroups x 32 (forced) -> 128 +// linear_dq8ca_q4gsw_coopmat 128x64x32, 2x2 subgroups x 64 -> 256 +// (The int8-MMA shaders stay on wave64: int8 WMMA at forced subgroup 32 +// crashes the Xclipse PAL compiler.) +struct CoopmatTileDims { + uint32_t m; + uint32_t n; + uint32_t k; + // Threads per workgroup = SG_GRID_X * SG_GRID_Y * SUBGROUP_SIZE. MUST match + // the WG_SIZE the shader yaml resolves to, or the launched thread count won't + // match the shader's staging passes (out-of-bounds). + uint32_t wg_size; +}; +// linear_qw_coopmat.yaml: 128x64, 2x2 subgroup grid, sg32 -> WG_SIZE 128. +constexpr CoopmatTileDims kQ4gswCoopmatDims = {128, 64, 16, 128}; +// linear_dq8ca_qw_coopmat.yaml: 128x64, 2x2 grid, sg64 -> WG_SIZE 256. +constexpr CoopmatTileDims kDq8caQ4gswCoopmatDims = {128, 64, 32, 256}; + +void resize_linear_qw_node( + ComputeGraph* graph, + const std::vector& args, + const std::vector& extra_args); + +GlobalWorkGrid quantized_linear_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args); + +LocalWorkGroup quantized_linear_lwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const GlobalWorkGrid& gwg, + const std::vector& args, + const std::vector& resize_args); + +bool can_use_q4gsw_coopmat( + ComputeGraph* graph, + const ValueRef output, + const ValueRef fp_input, + int64_t group_size, + const ValueRef bias, + int64_t tile_m = kCoopmatTileM, + int64_t tile_n = kCoopmatTileN, + int64_t tile_k = kCoopmatTileK); + +ValueRef prepack_quantized_linear_weight( + ComputeGraph& graph, + const QuantizationConfig& weight_quant_config, + const ValueRef qmat2_data, + const bool use_unsigned_dot = false); + +void quantized_linear_impl( + ComputeGraph& graph, + const QuantizationConfig& input_quant_config, + const QuantizationConfig& weight_quant_config, + const ValueRef fp_input, + const ValueRef input_scale, + const ValueRef input_zp, + const ValueRef weight_data, + const ValueRef weight_sums_data, + const ValueRef weight_scales_data, + const ValueRef weight_zeros_data, + const ValueRef group_size, + const ValueRef bias_data, + const ValueRef output); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.cpp new file mode 100644 index 00000000000..dd13ec2ebb5 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.cpp @@ -0,0 +1,192 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include + +#include +#include +#include +#include + +namespace vkcompute { + +vkapi::ShaderInfo pick_linear_dqa_qw_shader( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + + const ValueRef out = args.at(0).refs.at(0); + const ValueRef fp_input = args.at(1).refs.at(0); + const ValueRef int_input = args.at(1).refs.at(1); + (void)int_input; + const ValueRef input_zp = args.at(1).refs.at(4); + const ValueRef int_weight = args.at(1).refs.at(5); + + const bool weight_is_4bit = resize_args.at(0) != kDummyValueRef; + const bool is_gemv_case = is_gemv(graph, fp_input); + + // Use the coopmat shader for 4-bit dq8ca dispatches when the device + // enumerates VK_COMPONENT_TYPE_SINT8_KHR in its cooperative matrix property + // list and the shape aligns; tiled otherwise. + if (weight_is_4bit && !is_gemv_case && + graph->context()->adapter_ptr()->supports_int8_cooperative_matrix()) { + const int64_t group_size = + graph->extract_scalar(resize_args.at(0)); + if (can_use_q4gsw_coopmat( + graph, + out, + fp_input, + group_size, + resize_args.at(2), + kDq8caQ4gswCoopmatDims.m, + kDq8caQ4gswCoopmatDims.n, + kDq8caQ4gswCoopmatDims.k)) { + std::string kernel_name = "linear_dq8ca_q4gsw_coopmat"; + add_storage_type_suffix(kernel_name, graph->storage_type_of(out)); + add_storage_type_suffix(kernel_name, graph->storage_type_of(int_weight)); + add_dtype_suffix(kernel_name, graph->dtype_of(out)); + return VK_KERNEL_FROM_STR(kernel_name); + } + } + + std::string kernel_name = "linear_dq8ca_q4gsw"; + kernel_name += is_gemv_case ? "_coop" : "_tiled"; + add_storage_type_suffix(kernel_name, graph->storage_type_of(out)); + add_storage_type_suffix(kernel_name, graph->storage_type_of(int_weight)); + add_dtype_suffix(kernel_name, graph->dtype_of(out)); + add_zp_dtype_mode_suffix(kernel_name, graph->dtype_of(input_zp)); + + return VK_KERNEL_FROM_STR(kernel_name); +} + +void add_linear_dqa_qw_node( + ComputeGraph& graph, + const QuantizationConfig& input_quant_config, + const QuantizationConfig& weight_quant_config, + const ValueRef fp_input, + const ValueRef packed_int_input, + const ValueRef int_input_sums, + const ValueRef packed_input_scale, + const ValueRef packed_input_zp, + const ValueRef input_scale_data, + const ValueRef input_zp_data, + const ValueRef weight_data, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef group_size, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef output) { + (void)input_scale_data; + (void)input_zp_data; + VK_CHECK_COND(input_quant_config.granularity == kPerChannel); + VK_CHECK_COND(input_quant_config.nbits == 8); + VK_CHECK_COND(input_quant_config.is_dynamic); + + VK_CHECK_COND(weight_quant_config.granularity == kPerGroup); + VK_CHECK_COND(weight_quant_config.is_symmetric); + VK_CHECK_COND(weight_quant_config.nbits == 4); + + vkapi::ParamsBindList param_buffers = { + graph.sizes_ubo(output), graph.sizes_ubo(fp_input)}; + + uint32_t apply_bias = 1; + if (graph.val_is_none(bias_data)) { + apply_bias = 0; + } + + int32_t K4_per_group = 0; + int32_t coopmat_k_iters = 0; + const int32_t K_dim = graph.size_at(-1, fp_input); + if (weight_quant_config.nbits == 4) { + int32_t group_size_val = graph.extract_scalar(group_size); + K4_per_group = utils::div_up(group_size_val, int32_t(4)); + coopmat_k_iters = K_dim / group_size_val; + } + + const ValueRef is_4bit_flag = + weight_quant_config.nbits == 4 ? group_size : kDummyValueRef; + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + pick_linear_dqa_qw_shader, + quantized_linear_gwg, + quantized_linear_lwg, + // Inputs and Outputs + {{output, vkapi::kWrite}, + {{fp_input, + packed_int_input, + int_input_sums, + packed_input_scale, + packed_input_zp, + packed_weight, + packed_weight_sums, + packed_weight_scales, + packed_bias}, + vkapi::kRead}}, + // Shader params buffers + param_buffers, + // Push Constants + {}, + // Specialization Constants + // 4th spec const: output width N for coopMatStore (see + // add_linear_qw_node). + {apply_bias, + K4_per_group, + coopmat_k_iters, + graph.size_at(-1, output)}, + // Resize args (resize_args.at(2) = bias_data, read by the coopmat gate) + {is_4bit_flag, weight_data, bias_data}, + // Resizing Logic + resize_linear_qw_node)); +} + +void linear_dq8ca_q4gsw( + ComputeGraph& graph, + const std::vector& args) { + int32_t idx = 0; + const ValueRef fp_input = args.at(idx++); + const ValueRef input_scale = args.at(idx++); + const ValueRef input_zp = args.at(idx++); + const ValueRef weight_data = args.at(idx++); + const ValueRef weight_sums_data = args.at(idx++); + const ValueRef weight_scales_data = args.at(idx++); + const ValueRef group_size = args.at(idx++); + const ValueRef bias_data = args.at(idx++); + const ValueRef output = args.at(idx++); + + const int64_t group_size_val = graph.extract_scalar(group_size); + + QuantizationConfig input_quant_config(8, kPerChannel, {}, false, true); + QuantizationConfig weight_quant_config(4, kPerGroup, {group_size_val}); + + quantized_linear_impl( + graph, + input_quant_config, + weight_quant_config, + fp_input, + input_scale, + input_zp, + weight_data, + weight_sums_data, + weight_scales_data, + kDummyValueRef, // weight_zeros_data + group_size, // group_size + bias_data, + output); +} + +REGISTER_OPERATORS { + VK_REGISTER_OP(et_vk.linear_dq8ca_q4gsw.default, linear_dq8ca_q4gsw); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.h new file mode 100644 index 00000000000..0cff158a323 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/dq8ca_q4gsw/Dq8caQ4gswLinear.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +namespace vkcompute { + +void add_linear_dqa_qw_node( + ComputeGraph& graph, + const QuantizationConfig& input_quant_config, + const QuantizationConfig& weight_quant_config, + const ValueRef fp_input, + const ValueRef packed_int_input, + const ValueRef int_input_sums, + const ValueRef packed_input_scale, + const ValueRef packed_input_zp, + const ValueRef input_scale_data, + const ValueRef input_zp_data, + const ValueRef weight_data, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef group_size, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef output); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.cpp new file mode 100644 index 00000000000..c6c69be6a1c --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.cpp @@ -0,0 +1,128 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include + +#include +#include +#include + +namespace vkcompute { + +void add_linear_qa_qw_node( + ComputeGraph& graph, + const QuantizationConfig& input_quant_config, + const QuantizationConfig& weight_quant_config, + const ValueRef fp_input, + const ValueRef packed_int_input, + const ValueRef packed_input_scale, + const ValueRef packed_input_zp, + const ValueRef input_scale_data, + const ValueRef input_zp_data, + const ValueRef weight_data, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef group_size, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef output) { + VK_CHECK_COND(input_quant_config.granularity == kPerTensor); + VK_CHECK_COND(input_quant_config.nbits == 8); + VK_CHECK_COND(weight_quant_config.granularity == kPerChannel); + VK_CHECK_COND(weight_quant_config.is_symmetric); + VK_CHECK_COND(weight_quant_config.nbits == 8); + + float scale = graph.extract_scalar(input_scale_data); + int32_t zp = graph.extract_scalar(input_zp_data); + + // Get shader for quantized linear + std::string kernel_name = "linear_q8ta_q8csw_tiled"; + add_storage_type_suffix(kernel_name, graph.storage_type_of(output)); + add_storage_type_suffix(kernel_name, graph.storage_type_of(packed_int_input)); + add_storage_type_suffix(kernel_name, graph.storage_type_of(packed_weight)); + add_dtype_suffix(kernel_name, graph.dtype_of(output)); + vkapi::ShaderInfo shader = VK_KERNEL_FROM_STR(kernel_name); + + vkapi::ParamsBindList param_buffers = { + graph.sizes_ubo(output), graph.sizes_ubo(packed_int_input)}; + + std::vector push_constants = { + PushConstantDataInfo(&scale, sizeof(scale)), + PushConstantDataInfo(&zp, sizeof(zp)), + }; + + uint32_t apply_bias = 1; + if (graph.val_is_none(bias_data)) { + apply_bias = 0; + } + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + VK_KERNEL_FROM_STR(kernel_name), + quantized_linear_gwg, + quantized_linear_lwg, + // Inputs and Outputs + {{output, vkapi::kWrite}, + {{packed_int_input, + packed_weight, + packed_weight_sums, + packed_weight_scales, + packed_bias}, + vkapi::kRead}}, + // Shader params buffers + param_buffers, + // Push Constants + push_constants, + // Specialization Constants + {apply_bias}, + // Resize args + {fp_input}, + // Resizing Logic + nullptr)); +} + +void linear_q8ta_q8csw(ComputeGraph& graph, const std::vector& args) { + int32_t idx = 0; + const ValueRef fp_input = args.at(idx++); + const ValueRef input_scale = args.at(idx++); + const ValueRef input_zp = args.at(idx++); + const ValueRef weight_data = args.at(idx++); + const ValueRef weight_sums_data = args.at(idx++); + const ValueRef weight_scales_data = args.at(idx++); + const ValueRef bias_data = args.at(idx++); + const ValueRef output = args.at(idx++); + + const int64_t K = graph.size_at(-1, fp_input); + + QuantizationConfig input_quant_config(8, kPerTensor, {}, false); + QuantizationConfig weight_quant_config(8, kPerChannel, {K}); + + quantized_linear_impl( + graph, + input_quant_config, + weight_quant_config, + fp_input, + input_scale, + input_zp, + weight_data, + weight_sums_data, + weight_scales_data, + kDummyValueRef, // weight_zeros_data + kDummyValueRef, // group_size + bias_data, + output); +} + +REGISTER_OPERATORS { + VK_REGISTER_OP(et_vk.linear_q8ta_q8csw.default, linear_q8ta_q8csw); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.h new file mode 100644 index 00000000000..661da132b8c --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/Q8taQ8cswLinear.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include + +namespace vkcompute { + +void add_linear_qa_qw_node( + ComputeGraph& graph, + const QuantizationConfig& input_quant_config, + const QuantizationConfig& weight_quant_config, + const ValueRef fp_input, + const ValueRef packed_int_input, + const ValueRef packed_input_scale, + const ValueRef packed_input_zp, + const ValueRef input_scale_data, + const ValueRef input_zp_data, + const ValueRef weight_data, + const ValueRef packed_weight, + const ValueRef packed_weight_sums, + const ValueRef packed_weight_scales, + const ValueRef group_size, + const ValueRef bias_data, + const ValueRef packed_bias, + const ValueRef output); + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinearQCSNW.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/QuantizedLinearQCSNW.cpp similarity index 99% rename from backends/vulkan/runtime/graph/ops/impl/QuantizedLinearQCSNW.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/QuantizedLinearQCSNW.cpp index c2e45b96418..99b0fc1e225 100644 --- a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinearQCSNW.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/legacy/QuantizedLinearQCSNW.cpp @@ -298,7 +298,7 @@ void linear_qcs4w(ComputeGraph& graph, const std::vector& args) { } REGISTER_OPERATORS { - // aten._weight_int8pack_mm is registered in QuantizedLinear.cpp, on the + // aten._weight_int8pack_mm is registered in Q8cswLinear.cpp, on the // maintained weight-only quantized linear implementation. The 8-bit path // here dispatched to linear_qcs8w_* shaders that no longer exist. VK_REGISTER_OP(et_vk.linear_qcs4w.default, linear_qcs4w); diff --git a/backends/vulkan/runtime/graph/ops/impl/Q4gswLinear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/Q4gswLinear.cpp similarity index 99% rename from backends/vulkan/runtime/graph/ops/impl/Q4gswLinear.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/Q4gswLinear.cpp index cf8afc8c8c1..8e7d16092c7 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Q4gswLinear.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/Q4gswLinear.cpp @@ -6,7 +6,7 @@ * LICENSE file in the root directory of this source tree. */ -#include +#include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/Q4gswLinear.h b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/Q4gswLinear.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/Q4gswLinear.h rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/Q4gswLinear.h diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinearBackward.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/backward/QuantizedLinearBackward.cpp similarity index 97% rename from backends/vulkan/runtime/graph/ops/impl/QuantizedLinearBackward.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/backward/QuantizedLinearBackward.cpp index 8bfe1681bbf..01ebb2f692d 100644 --- a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinearBackward.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/backward/QuantizedLinearBackward.cpp @@ -11,7 +11,7 @@ #include #include -#include +#include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizedLinearRequant.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/backward/QuantizedLinearRequant.cpp similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/QuantizedLinearRequant.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q4gsw/backward/QuantizedLinearRequant.cpp diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q8csw/Q8cswLinear.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q8csw/Q8cswLinear.cpp new file mode 100644 index 00000000000..89657534b9b --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/linear/qw/q8csw/Q8cswLinear.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include + +namespace vkcompute { + +void linear_q8csw(ComputeGraph& graph, const std::vector& args) { + int32_t idx = 0; + const ValueRef fp_input = args.at(idx++); + const ValueRef weight_data = args.at(idx++); + const ValueRef weight_scales_data = args.at(idx++); + const ValueRef bias_data = args.at(idx++); + const ValueRef output = args.at(idx++); + + const int64_t K = graph.size_at(-1, fp_input); + + QuantizationConfig input_quant_config(32, kNoQuantization, {}); + QuantizationConfig weight_quant_config(8, kPerChannel, {K}); + + quantized_linear_impl( + graph, + input_quant_config, + weight_quant_config, + fp_input, + kDummyValueRef, // input scale + kDummyValueRef, // input zp + weight_data, + kDummyValueRef, // weight sums + weight_scales_data, + kDummyValueRef, // weight zeros + kDummyValueRef, // group size + bias_data, + output); +} + +// aten._weight_int8pack_mm is what the AOT weight-only int8 fusion +// (FuseQuantizedOpsTransform) emits. It carries the same operands as +// et_vk.linear_q8csw minus the bias, so it runs through the same +// implementation. +void weight_int8pack_mm( + ComputeGraph& graph, + const std::vector& args) { + int32_t idx = 0; + const ValueRef fp_input = args.at(idx++); + const ValueRef weight_data = args.at(idx++); + const ValueRef weight_scales_data = args.at(idx++); + const ValueRef output = args.at(idx++); + + const int64_t K = graph.size_at(-1, fp_input); + + QuantizationConfig input_quant_config(32, kNoQuantization, {}); + QuantizationConfig weight_quant_config(8, kPerChannel, {K}); + + quantized_linear_impl( + graph, + input_quant_config, + weight_quant_config, + fp_input, + kDummyValueRef, // input scale + kDummyValueRef, // input zp + weight_data, + kDummyValueRef, // weight sums + weight_scales_data, + kDummyValueRef, // weight zeros + kDummyValueRef, // group size + kDummyValueRef, // bias + output); +} + +REGISTER_OPERATORS { + VK_REGISTER_OP(et_vk.linear_q8csw.default, linear_q8csw); + VK_REGISTER_OP(aten._weight_int8pack_mm.default, weight_int8pack_mm); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/Matmul.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatMul.cpp similarity index 95% rename from backends/vulkan/runtime/graph/ops/impl/Matmul.cpp rename to backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatMul.cpp index fc86255be51..e314436cc7c 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Matmul.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatMul.cpp @@ -9,11 +9,12 @@ #include #include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include #include #include diff --git a/backends/vulkan/runtime/graph/ops/impl/MatMul.h b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatMul.h similarity index 100% rename from backends/vulkan/runtime/graph/ops/impl/MatMul.h rename to backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatMul.h diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.cpp b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.cpp new file mode 100644 index 00000000000..1cc9730ac9b --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include +#include + +#include + +namespace vkcompute { + +static vkapi::ShaderInfo pick_matmul_coopmat_shader( + ComputeGraph* graph, + const std::vector& args, + const std::vector& resize_args) { + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + std::string kernel_name = "matmul_coopmat"; + kernel_name.reserve(kShaderNameReserve); + add_dtype_suffix(kernel_name, graph->dtype_of(out)); + return VK_KERNEL_FROM_STR(kernel_name); +} + +static GlobalWorkGrid pick_matmul_coopmat_gwg( + ComputeGraph* graph, + const vkapi::ShaderInfo& shader, + const std::vector& args, + const std::vector& resize_args) { + (void)shader; + (void)resize_args; + const ValueRef out = args.at(0).refs.at(0); + const auto out_sizes = graph->sizes_of(out); + uint32_t M = + utils::safe_downcast(out_sizes.at(out_sizes.size() - 2)); + uint32_t N = + utils::safe_downcast(out_sizes.at(out_sizes.size() - 1)); + uint32_t num_tiles_n = utils::div_up(N, kCoopmatTileN); + uint32_t num_tiles_m = utils::div_up(M, kCoopmatTileM); + // Each workgroup processes one WG_TILE_M x WG_TILE_N output tile via + // cooperative-matrix MMAs across its 4 subgroups. We want the dispatch + // to launch exactly num_tiles_n x num_tiles_m workgroups. + // + // The framework computes the group count as + // group_count = div_up(gwg, lwg) + // (see Context.cpp + Command.cpp). With lwg = (kCoopmatInvocations, + // 1, 1), multiplying num_tiles_n by kCoopmatInvocations cancels the + // div, yielding group_count.x = num_tiles_n. + return GlobalWorkGrid( + {num_tiles_n * kCoopmatInvocations, num_tiles_m, 1u}, + kTiledWorkGrid, + LocalWorkGroup(kCoopmatInvocations, 1u, 1u)); +} + +void add_matmul_coopmat_node( + ComputeGraph& graph, + const ValueRef mat1, + const ValueRef mat2, + const ValueRef out) { + VK_CHECK_COND(graph.packed_dim_of(mat1) == WHCN::kWidthDim); + VK_CHECK_COND(graph.packed_dim_of(mat2) == WHCN::kWidthDim); + VK_CHECK_COND(graph.packed_dim_of(out) == WHCN::kWidthDim); + VK_CHECK_COND( + graph.storage_type_of(out) == utils::kBuffer, + "matmul_coopmat requires buffer storage"); + + ValueRef has_bias_ref = graph.add_scalar(false); + + graph.execute_nodes().emplace_back(new DynamicDispatchNode( + graph, + pick_matmul_coopmat_shader, + pick_matmul_coopmat_gwg, + pick_required_lwg, + // Inputs and Outputs — same binding order as matmul_vec + {{out, vkapi::kWrite}, {{mat1, mat2}, vkapi::kRead}}, + // Shader params buffers — same UBOs as matmul_vec + {graph.sizes_ubo(mat1), graph.sizes_ubo(mat2)}, + // Push Constants + {}, + // Specialization Constants + {}, + // Resize Args + {has_bias_ref}, + // Resizing Logic + resize_matmul_tiled_node)); +} + +} // namespace vkcompute diff --git a/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.h b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.h new file mode 100644 index 00000000000..4cf825fc838 --- /dev/null +++ b/backends/vulkan/runtime/graph/ops/impl/gemm/matmul/MatmulCoopmat.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include + +namespace vkcompute { + +void add_matmul_coopmat_node( + ComputeGraph& graph, + const ValueRef mat1, + const ValueRef mat2, + const ValueRef out); + +} // namespace vkcompute diff --git a/backends/vulkan/targets.bzl b/backends/vulkan/targets.bzl index d2dee601399..aaa96eccb63 100644 --- a/backends/vulkan/targets.bzl +++ b/backends/vulkan/targets.bzl @@ -184,7 +184,7 @@ def define_common_targets(is_fbcode = False): runtime.filegroup( name = "vulkan_graph_runtime_shaders", srcs = native.glob([ - "runtime/graph/ops/glsl/*", + "runtime/graph/ops/glsl/**/*", ]), ) diff --git a/backends/vulkan/test/custom_ops/q8ta_conv2d_route_test.cpp b/backends/vulkan/test/custom_ops/q8ta_conv2d_route_test.cpp index 63d154b170f..c660430b579 100644 --- a/backends/vulkan/test/custom_ops/q8ta_conv2d_route_test.cpp +++ b/backends/vulkan/test/custom_ops/q8ta_conv2d_route_test.cpp @@ -4,7 +4,7 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include +#include #include diff --git a/backends/vulkan/test/custom_ops/q8ta_conv2d_stream_plan_test.cpp b/backends/vulkan/test/custom_ops/q8ta_conv2d_stream_plan_test.cpp index 6f634a83c27..c1207cb1923 100644 --- a/backends/vulkan/test/custom_ops/q8ta_conv2d_stream_plan_test.cpp +++ b/backends/vulkan/test/custom_ops/q8ta_conv2d_stream_plan_test.cpp @@ -4,7 +4,7 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include +#include #include diff --git a/backends/vulkan/test/custom_ops/targets.bzl b/backends/vulkan/test/custom_ops/targets.bzl index 1895180495a..2355e6a4f2a 100644 --- a/backends/vulkan/test/custom_ops/targets.bzl +++ b/backends/vulkan/test/custom_ops/targets.bzl @@ -1,90 +1,14 @@ -load("@fbsource//tools/build_defs:platform_defs.bzl", "ANDROID", "CXX") +load("@fbsource//tools/build_defs:platform_defs.bzl", "CXX") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") load( "@fbsource//xplat/executorch/backends/vulkan:targets.bzl", "get_platforms", - "vulkan_spv_shader_lib", ) -def define_custom_op_test_binary(custom_op_name, extra_deps = [], src_file = None, include_torch = False): - deps_list = [ - ":prototyping_utils", - ":operator_implementations", - ":custom_ops_shaderlib", - "//executorch/backends/vulkan:vulkan_graph_runtime", - ] + ([runtime.external_dep_location("libtorch")] if include_torch else []) + extra_deps - - src_file_str = src_file if src_file else "{}.cpp".format(custom_op_name) - - runtime.cxx_binary( - name = custom_op_name, - srcs = [ - src_file_str, - ], - platforms = get_platforms(), - define_static_target = False, - deps = deps_list, - ) - def define_common_targets(is_fbcode = False): if is_fbcode: return - # Shader library from GLSL files - runtime.filegroup( - name = "custom_ops_shaders", - srcs = native.glob([ - "glsl/*.glsl", - "glsl/*.yaml", - ]), - visibility = ["PUBLIC"], - ) - - vulkan_spv_shader_lib( - name = "custom_ops_shaderlib", - spv_filegroups = { - ":custom_ops_shaders": "glsl", - }, - is_fbcode = is_fbcode, - ) - - # Prototyping utilities library - runtime.cxx_library( - name = "prototyping_utils", - srcs = [ - "utils.cpp", - "conv2d_utils.cpp", - ], - headers = [ - "utils.h", - "conv2d_utils.h", - ], - exported_headers = [ - "utils.h", - "conv2d_utils.h", - ], - platforms = get_platforms(), - deps = [ - "//executorch/backends/vulkan:vulkan_graph_runtime", - ], - visibility = ["PUBLIC"], - ) - - # Operator implementations library - runtime.cxx_library( - name = "operator_implementations", - srcs = native.glob([ - "impl/*.cpp", - ]), - platforms = get_platforms(), - deps = [ - "//executorch/backends/vulkan:vulkan_graph_runtime", - ":custom_ops_shaderlib", - ], - visibility = ["PUBLIC"], - link_whole = True, - ) - runtime.cxx_test( name = "utils_test", srcs = [ @@ -93,7 +17,7 @@ def define_common_targets(is_fbcode = False): contacts = ["oncall+ai_infra_mobile_platform@xmail.facebook.com"], platforms = [CXX], deps = [ - ":prototyping_utils", + "//executorch/backends/vulkan/test/ops/benchmark:prototyping_utils", "//third-party/googletest:gtest_main", ], ) @@ -117,28 +41,3 @@ def define_common_targets(is_fbcode = False): "//executorch/backends/vulkan:vulkan_graph_runtime", ], ) - - define_custom_op_test_binary("test_add") - define_custom_op_test_binary("test_q8csw_linear") - define_custom_op_test_binary("test_q8csw_conv2d") - define_custom_op_test_binary("test_choose_qparams_per_row") - define_custom_op_test_binary("test_q4gsw_linear") - define_custom_op_test_binary("test_q8ta_qdq") - define_custom_op_test_binary("test_q8ta_clone") - define_custom_op_test_binary("test_q8ta_binary") - define_custom_op_test_binary("test_q8ta_conv2d") - define_custom_op_test_binary("test_q8ta_conv2d_pw") - define_custom_op_test_binary("test_q8ta_conv2d_dw") - define_custom_op_test_binary("test_q8ta_linear") - define_custom_op_test_binary("test_q8ta_conv2d_transposed") - define_custom_op_test_binary("test_q8ta_pixel_shuffle") - define_custom_op_test_binary("test_q8ta_unary") - define_custom_op_test_binary("test_mm") - define_custom_op_test_binary("test_conv2d") - define_custom_op_test_binary("test_conv2d_pw") - define_custom_op_test_binary("test_conv2d_dw") - define_custom_op_test_binary("test_embedding_q4gsw") - define_custom_op_test_binary("test_conv1d_pw") - define_custom_op_test_binary("test_conv1d_dw") - define_custom_op_test_binary("test_fpa_q4gsw_linear") - define_custom_op_test_binary("test_sdpa") diff --git a/backends/vulkan/test/custom_ops/test_add.cpp b/backends/vulkan/test/custom_ops/test_add.cpp deleted file mode 100644 index bd7c644c2fd..00000000000 --- a/backends/vulkan/test/custom_ops/test_add.cpp +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include -#include -#include -#include "utils.h" - -using namespace executorch::vulkan::prototyping; - -// Generate test cases for add operation -std::vector generate_add_test_cases() { - std::vector test_cases; - - // Set the data generation type as a local variable - DataGenType data_gen_type = DataGenType::ONES; - - // Define different input size configurations - std::vector> size_configs = { - {1, 64, 64}, // Small square - {1, 128, 128}, // Medium square - {1, 256, 256}, // Large square - {1, 512, 512}, // Very large square - {1, 1, 1024}, // Wide tensor - {1, 1024, 1}, // Tall tensor - {32, 32, 32}, // 3D cube - {16, 128, 64}, // 3D rectangular - }; - - // Storage types to test - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - // Data types to test - std::vector data_types = {vkapi::kFloat, vkapi::kHalf}; - - // Generate test cases for each combination - for (const auto& sizes : size_configs) { - for (const auto& storage_type : storage_types) { - for (const auto& data_type : data_types) { - TestCase test_case; - - // Create a descriptive name for the test case - std::string shape_str = - shape_bracket(sizes) + "+" + shape_bracket(sizes); - std::string storage_str = repr_str(storage_type, utils::kWidthPacked); - std::string dtype_str = dtype_short(data_type); - std::string test_name = make_test_label( - "ACCU", dtype_str, dtype_str, shape_str, storage_str); - test_case.set_name(test_name); - - // Set the operator name for the test case - test_case.set_operator_name("etvk.add_prototype"); - - // Add two input tensors with the same size, type, storage, and data - // generation method - ValueSpec input_a( - sizes, data_type, storage_type, utils::kWidthPacked, data_gen_type); - ValueSpec input_b( - sizes, data_type, storage_type, utils::kWidthPacked, data_gen_type); - - // Add output tensor with the same size, type, and storage as inputs - // (output uses ZEROS by default) - ValueSpec output( - sizes, - data_type, - storage_type, - utils::kWidthPacked, - DataGenType::ZEROS); - - test_case.add_input_spec(input_a); - test_case.add_input_spec(input_b); - test_case.add_output_spec(output); - - test_cases.push_back(test_case); - } - } - } - - return test_cases; -} - -// Custom FLOP calculator for add operation -// Add operation performs 1 FLOP (addition) per element -int64_t add_flop_calculator(const TestCase& test_case) { - // Calculate total elements from the first input tensor - int64_t total_elements = 1; - if (!test_case.empty() && test_case.num_inputs() > 0 && - test_case.inputs()[0].is_tensor()) { - const auto& sizes = test_case.inputs()[0].get_tensor_sizes(); - for (int64_t size : sizes) { - total_elements *= size; - } - } - - // Add operation: 1 FLOP per element (one addition) - return total_elements; -} - -// Reference implementation for add operator -void add_reference_compute(TestCase& test_case) { - const ValueSpec& input_a = test_case.inputs().at(0); - const ValueSpec& input_b = test_case.inputs().at(1); - - ValueSpec& output = test_case.outputs().at(0); - - if (input_a.dtype != vkapi::kFloat) { - throw std::invalid_argument("Unsupported dtype"); - } - - // Calculate number of elements - int64_t num_elements = input_a.numel(); - - auto& input_a_data = input_a.get_float_data(); - auto& input_b_data = input_b.get_float_data(); - - auto& ref_data = output.get_ref_float_data(); - ref_data.resize(num_elements); - for (int64_t i = 0; i < num_elements; ++i) { - ref_data[i] = input_a_data[i] + input_b_data[i]; - } -} - -int main(int argc, char* argv[]) { - set_print_output(false); // Disable output tensor printing - set_print_latencies(false); // Enable latency timing printing - set_use_gpu_timestamps(true); // Enable GPU timestamps - - print_performance_header(); - std::cout << "Add Operation Prototyping Framework" << std::endl; - print_separator(); - - // Initialize Vulkan context - try { - api::context()->initialize_querypool(); - } catch (const std::exception& e) { - std::cerr << "Failed to initialize Vulkan context: " << e.what() - << std::endl; - return 1; - } - - // Execute test cases using the new framework with custom FLOP calculator and - // reference compute - auto results = execute_test_cases( - generate_add_test_cases, - add_flop_calculator, - "Add", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - add_reference_compute); - - return 0; -} diff --git a/backends/vulkan/test/custom_ops/test_conv1d_pw.cpp b/backends/vulkan/test/custom_ops/test_conv1d_pw.cpp deleted file mode 100644 index dc40d1ee795..00000000000 --- a/backends/vulkan/test/custom_ops/test_conv1d_pw.cpp +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include - -#include -#include - -#include "utils.h" - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 256; - -struct Conv1dPWConfig { - int64_t N; - int64_t C_in; - int64_t C_out; - int64_t L; - bool has_bias; -}; - -static TestCase create_conv1d_pw_test_case( - const Conv1dPWConfig& config, - vkapi::ScalarType dtype, - utils::StorageType storage_type) { - TestCase test_case; - - bool is_perf = config.C_in > kRefDimSizeLimit || - config.C_out > kRefDimSizeLimit || config.L > kRefDimSizeLimit; - - std::string prefix = is_perf ? "PERF" : "ACCU"; - std::string storage_str = storage_type_abbrev(storage_type) + "(HP)"; - std::string dtype_str = dtype_short(dtype); - - std::string bias_str = config.has_bias ? "+bias" : ""; - - std::string shape = "[" + std::to_string(config.N) + "," + - std::to_string(config.C_in) + "," + std::to_string(config.L) + "]x[" + - std::to_string(config.C_out) + "," + std::to_string(config.C_in) + ",1]"; - - std::string name = make_test_label( - prefix, dtype_str, dtype_str, shape, storage_str, bias_str); - - test_case.set_name(name); - test_case.set_operator_name("test_etvk.test_conv1d_pw.default"); - - // Input: [N, C_in, L] height-packed - ValueSpec input( - {config.N, config.C_in, config.L}, - dtype, - storage_type, - utils::kHeightPacked, - DataGenType::RANDOM); - test_case.add_input_spec(input); - - // Weight: [C_out, C_in, 1] height-packed, constant - ValueSpec weight( - {config.C_out, config.C_in, 1}, - dtype, - storage_type, - utils::kHeightPacked, - DataGenType::RANDOM); - weight.set_constant(true); - test_case.add_input_spec(weight); - - // Bias: [C_out] or None - if (config.has_bias) { - ValueSpec bias( - {config.C_out}, - dtype, - storage_type, - utils::kWidthPacked, - DataGenType::RANDOM); - bias.set_constant(true); - test_case.add_input_spec(bias); - } else { - ValueSpec none_bias(static_cast(0)); - none_bias.set_none(true); - test_case.add_input_spec(none_bias); - } - - // stride = [1] - test_case.add_input_spec(ValueSpec(std::vector{1})); - // padding = [0] - test_case.add_input_spec(ValueSpec(std::vector{0})); - // dilation = [1] - test_case.add_input_spec(ValueSpec(std::vector{1})); - // groups = 1 - test_case.add_input_spec(ValueSpec(static_cast(1))); - - // Output: [N, C_out, L] height-packed - ValueSpec output( - {config.N, config.C_out, config.L}, - dtype, - storage_type, - utils::kHeightPacked, - DataGenType::ZEROS); - test_case.add_output_spec(output); - - if (dtype == vkapi::kHalf) { - test_case.set_abs_tolerance(1e-1f); - test_case.set_rel_tolerance(1e-2f); - } else { - test_case.set_abs_tolerance(1e-3f); - test_case.set_rel_tolerance(1e-3f); - } - - test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); - - return test_case; -} - -static void conv1d_pw_reference_impl(TestCase& test_case) { - const auto& input_spec = test_case.inputs()[0]; - const auto& weight_spec = test_case.inputs()[1]; - const auto& bias_spec = test_case.inputs()[2]; - ValueSpec& output = test_case.outputs()[0]; - - if (input_spec.dtype != vkapi::kFloat) { - throw std::invalid_argument("Reference only supports float"); - } - - auto in_sizes = input_spec.get_tensor_sizes(); - auto w_sizes = weight_spec.get_tensor_sizes(); - - int64_t N = in_sizes[0]; - int64_t C_in = in_sizes[1]; - int64_t L = in_sizes[2]; - int64_t C_out = w_sizes[0]; - - const auto& in_data = input_spec.get_float_data(); - const auto& w_data = weight_spec.get_float_data(); - auto& ref_data = output.get_ref_float_data(); - ref_data.resize(N * C_out * L, 0.0f); - - // input is NCHW-contiguous: [N, C_in, L] - // weight is [C_out, C_in, 1] - for (int64_t n = 0; n < N; ++n) { - for (int64_t oc = 0; oc < C_out; ++oc) { - for (int64_t l = 0; l < L; ++l) { - float sum = 0.0f; - for (int64_t ic = 0; ic < C_in; ++ic) { - sum += in_data[n * C_in * L + ic * L + l] * w_data[oc * C_in + ic]; - } - ref_data[n * C_out * L + oc * L + l] = sum; - } - } - } - - if (!bias_spec.is_none()) { - const auto& bias_data = bias_spec.get_float_data(); - for (int64_t n = 0; n < N; ++n) { - for (int64_t oc = 0; oc < C_out; ++oc) { - for (int64_t l = 0; l < L; ++l) { - ref_data[n * C_out * L + oc * L + l] += bias_data[oc]; - } - } - } - } -} - -static std::vector generate_conv1d_pw_test_cases() { - std::vector test_cases; - - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - // Accuracy shapes (float, small) - std::vector accu_configs = { - {1, 16, 32, 64, false}, - {1, 16, 32, 64, true}, - {1, 32, 16, 128, false}, - {1, 32, 16, 128, true}, - {1, 64, 64, 32, false}, - {1, 128, 256, 16, true}, - {2, 16, 32, 64, false}, - {2, 16, 32, 64, true}, - // Non-aligned channel counts (not a multiple of 4) - {1, 5, 7, 64, false}, - {1, 5, 7, 64, true}, - {1, 13, 17, 48, false}, - {1, 13, 17, 48, true}, - {1, 7, 5, 32, false}, - {2, 5, 13, 64, true}, - }; - - for (const auto& cfg : accu_configs) { - for (auto st : storage_types) { - test_cases.push_back(create_conv1d_pw_test_case(cfg, vkapi::kFloat, st)); - } - } - - // Performance shapes (half + float) - std::vector perf_configs = { - {1, 256, 512, 1024, false}, - {1, 256, 512, 1024, true}, - {1, 512, 256, 2048, false}, - {1, 128, 128, 4096, true}, - }; - - for (const auto& cfg : perf_configs) { - for (auto st : storage_types) { - test_cases.push_back(create_conv1d_pw_test_case(cfg, vkapi::kFloat, st)); - test_cases.push_back(create_conv1d_pw_test_case(cfg, vkapi::kHalf, st)); - } - } - - return test_cases; -} - -static int64_t conv1d_pw_flop_calculator(const TestCase& test_case) { - auto in_sizes = test_case.inputs()[0].get_tensor_sizes(); - auto w_sizes = test_case.inputs()[1].get_tensor_sizes(); - - int64_t N = in_sizes[0]; - int64_t C_in = in_sizes[1]; - int64_t L = in_sizes[2]; - int64_t C_out = w_sizes[0]; - - return 2 * N * C_in * C_out * L; -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Conv1d Pointwise (Height-Packed) Benchmark" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = conv1d_pw_reference_impl; - - auto results = execute_test_cases( - generate_conv1d_pw_test_cases, - conv1d_pw_flop_calculator, - "Conv1dPW", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/custom_ops/test_conv2d.cpp b/backends/vulkan/test/custom_ops/test_conv2d.cpp deleted file mode 100644 index 6bec62a4e61..00000000000 --- a/backends/vulkan/test/custom_ops/test_conv2d.cpp +++ /dev/null @@ -1,639 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include - -#include -#include - -#include "conv2d_utils.h" -#include "utils.h" - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 64; - -struct InputDims { - int64_t N; - int64_t C; - int64_t H; - int64_t W; - - InputDims(int64_t n, int64_t c, int64_t h, int64_t w) - : N(n), C(c), H(h), W(w) {} -}; - -struct Conv2dTestConfig { - InputDims dims; - int64_t C_out; - KernelSize kernel; - Stride stride; - Padding padding; - Dilation dilation; - bool has_bias; -}; - -static int64_t calc_out_size( - int64_t in_size, - int64_t kernel_size, - int64_t stride, - int64_t padding, - int64_t dilation) { - return (in_size + 2 * padding - dilation * (kernel_size - 1) - 1) / stride + - 1; -} - -// Shared perf/skip classification used by both create_conv2d_test_case (to tag -// PERF vs ACCU) and conv2d_reference_impl (to gate the large-K FP16 reference -// check). A shape is "perf" if any dimension reaches kRefDimSizeLimit; the -// boundary is inclusive (>=) so a 64-wide dim counts as perf — FP16 -// accumulation error at K = K_h * K_w * C_in for such shapes can exceed the -// half tolerance and false-fail. Keep both call sites on this single helper to -// avoid the two predicates drifting apart. -static bool -conv2d_is_perf_shape(int64_t C_in, int64_t C_out, int64_t H, int64_t W) { - return C_in >= kRefDimSizeLimit || C_out >= kRefDimSizeLimit || - H >= kRefDimSizeLimit || W >= kRefDimSizeLimit; -} - -static TestCase create_conv2d_test_case( - const Conv2dTestConfig& config, - vkapi::ScalarType dtype, - utils::StorageType storage_type, - utils::GPUMemoryLayout memory_layout, - const std::string& impl_selector = "") { - TestCase test_case; - - bool is_perf = conv2d_is_perf_shape( - config.dims.C, config.C_out, config.dims.H, config.dims.W); - - std::string prefix = is_perf ? "PERF" : "ACCU"; - std::string storage_str = repr_str(storage_type, memory_layout); - std::string dtype_str = dtype_short(dtype); - std::string bias_str = config.has_bias ? "+bias" : ""; - - int64_t H_out = calc_out_size( - config.dims.H, - config.kernel.h, - config.stride.h, - config.padding.h, - config.dilation.h); - int64_t W_out = calc_out_size( - config.dims.W, - config.kernel.w, - config.stride.w, - config.padding.w, - config.dilation.w); - - std::string shape = "[" + std::to_string(config.dims.N) + "," + - std::to_string(config.dims.C) + "," + std::to_string(config.dims.H) + - "," + std::to_string(config.dims.W) + "]->[" + - std::to_string(config.C_out) + "] k" + std::to_string(config.kernel.h) + - "x" + std::to_string(config.kernel.w) + " s" + - std::to_string(config.stride.h) + " p" + - std::to_string(config.padding.h) + " d" + - std::to_string(config.dilation.h); - - std::string suffix = bias_str; - if (!impl_selector.empty()) { - if (!suffix.empty()) { - suffix += " "; - } - suffix += "[" + impl_selector + "]"; - } - - std::string name = - make_test_label(prefix, dtype_str, dtype_str, shape, storage_str, suffix); - - test_case.set_name(name); - test_case.set_operator_name("test_etvk.test_conv2d.default"); - - // Input tensor [N, C_in, H, W] - ValueSpec input( - {config.dims.N, config.dims.C, config.dims.H, config.dims.W}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - - // Weight tensor [C_out, C_in, K_h, K_w] - constant - ValueSpec weight( - {config.C_out, config.dims.C, config.kernel.h, config.kernel.w}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - weight.set_constant(true); - - test_case.add_input_spec(input); - test_case.add_input_spec(weight); - - // Bias (or none) - if (config.has_bias) { - ValueSpec bias( - {config.C_out}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - bias.set_constant(true); - test_case.add_input_spec(bias); - } else { - ValueSpec none_bias(static_cast(0)); - none_bias.set_none(true); - test_case.add_input_spec(none_bias); - } - - // stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w - test_case.add_input_spec(ValueSpec(static_cast(config.stride.h))); - test_case.add_input_spec(ValueSpec(static_cast(config.stride.w))); - test_case.add_input_spec(ValueSpec(static_cast(config.padding.h))); - test_case.add_input_spec(ValueSpec(static_cast(config.padding.w))); - test_case.add_input_spec(ValueSpec(static_cast(config.dilation.h))); - test_case.add_input_spec(ValueSpec(static_cast(config.dilation.w))); - - // impl_selector string - test_case.add_input_spec(ValueSpec::make_string(impl_selector)); - - // Output tensor [N, C_out, H_out, W_out] - ValueSpec output( - {config.dims.N, config.C_out, H_out, W_out}, - dtype, - storage_type, - memory_layout, - DataGenType::ZEROS); - test_case.add_output_spec(output); - - if (dtype == vkapi::kHalf) { - test_case.set_abs_tolerance(1e-1f); - test_case.set_rel_tolerance(1e-2f); - } else { - test_case.set_abs_tolerance(1e-3f); - test_case.set_rel_tolerance(1e-3f); - } - - test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); - - return test_case; -} - -// Reference implementation for general conv2d (groups=1). -// -// Supports both FP32 and (small-shape) FP16 inputs. The math is always done in -// float; for FP16 the master input/weight/bias values are dequantized from -// their half storage via get_element(), and the resulting float reference is -// compared against the dequantized GPU output by validate_against_reference(). -// -// FP16 accumulation error grows with K (= K_h * K_w * C_in). For large-K PERF -// shapes the FP32 reference would diverge from the GPU's FP16 accumulation -// enough to trip even the relaxed half tolerance, producing false failures, so -// those are intentionally left timing-only: this function throws -// std::invalid_argument, which execute_test_cases() catches to skip the -// correctness check (ref_computed stays false) while still benchmarking. -static void conv2d_reference_impl(TestCase& test_case) { - const ValueSpec& input = test_case.inputs()[0]; - const ValueSpec& weight = test_case.inputs()[1]; - const ValueSpec& bias_spec = test_case.inputs()[2]; - ValueSpec& output = test_case.outputs()[0]; - - if (input.dtype != vkapi::kFloat && input.dtype != vkapi::kHalf) { - throw std::invalid_argument("Reference only supports float and half"); - } - - auto input_sizes = input.get_tensor_sizes(); - auto weight_sizes = weight.get_tensor_sizes(); - auto output_sizes = output.get_tensor_sizes(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t H_in = input_sizes[2]; - int64_t W_in = input_sizes[3]; - int64_t C_out = weight_sizes[0]; - int64_t K_h = weight_sizes[2]; - int64_t K_w = weight_sizes[3]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - // For FP16, only compute a reference for small (ACCU) shapes where K is small - // enough that FP32-vs-FP16 accumulation error stays within the half - // tolerance. Large-K PERF half shapes stay timing-only via the throw below. - // The predicate mirrors create_conv2d_test_case's is_perf classification. - if (input.dtype == vkapi::kHalf) { - const bool is_perf = conv2d_is_perf_shape(C_in, C_out, H_in, W_in); - if (is_perf) { - throw std::invalid_argument( - "Half reference skipped for large-K PERF shape (timing-only)"); - } - } - - int64_t stride_h = test_case.inputs()[3].get_int_value(); - int64_t stride_w = test_case.inputs()[4].get_int_value(); - int64_t padding_h = test_case.inputs()[5].get_int_value(); - int64_t padding_w = test_case.inputs()[6].get_int_value(); - int64_t dilation_h = test_case.inputs()[7].get_int_value(); - int64_t dilation_w = test_case.inputs()[8].get_int_value(); - - // get_element() materializes a float regardless of dtype (it dequantizes - // half master data), so the same loop body serves both FP32 and FP16. - auto& ref_data = output.get_ref_float_data(); - ref_data.resize(N * C_out * H_out * W_out, 0.0f); - - for (int64_t n = 0; n < N; ++n) { - for (int64_t co = 0; co < C_out; ++co) { - for (int64_t oh = 0; oh < H_out; ++oh) { - for (int64_t ow = 0; ow < W_out; ++ow) { - float sum = 0.0f; - for (int64_t ci = 0; ci < C_in; ++ci) { - for (int64_t kh = 0; kh < K_h; ++kh) { - for (int64_t kw = 0; kw < K_w; ++kw) { - int64_t ih = oh * stride_h - padding_h + kh * dilation_h; - int64_t iw = ow * stride_w - padding_w + kw * dilation_w; - if (ih >= 0 && ih < H_in && iw >= 0 && iw < W_in) { - float in_val = input.get_element( - n * (C_in * H_in * W_in) + ci * (H_in * W_in) + - ih * W_in + iw); - // weight is [C_out, C_in, K_h, K_w] - float w_val = weight.get_element( - co * (C_in * K_h * K_w) + ci * (K_h * K_w) + kh * K_w + - kw); - sum += in_val * w_val; - } - } - } - } - if (!bias_spec.is_none()) { - sum += bias_spec.get_element(co); - } - ref_data - [n * (C_out * H_out * W_out) + co * (H_out * W_out) + oh * W_out + - ow] = sum; - } - } - } - } -} - -static std::vector generate_conv2d_test_cases() { - std::vector test_cases; - - std::vector storage_types = {utils::kTexture3D}; - utils::GPUMemoryLayout layout = utils::kChannelsPacked; - - // Accuracy shapes (small enough for float reference validation) - std::vector accuracy_configs = { - // 3x3 stride=1 pad=1 same-channels (the bottleneck pattern in TinyCNN) - {InputDims(1, 8, 8, 8), - 8, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 8, 8, 8), - 8, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - {InputDims(1, 16, 16, 16), - 16, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // 3x3 stride=2 (downsample) with channel expansion - {InputDims(1, 8, 16, 16), - 16, - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - // 3x3 stride=1 with channel reduction - {InputDims(1, 16, 8, 8), - 8, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - // Non-multiple-of-4 channels - {InputDims(1, 11, 8, 8), - 13, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - // 3-channel input (like RGB stem) - {InputDims(1, 3, 16, 16), - 8, - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - }; - - // TinyCNN depth estimator hotspots (from profiling - // UNTRAINED_TinyCNNDepthEstimatorRealTime_Vulkan.pte). - // Each entry lists (C_in, H, W) -> C_out, all 3x3 stride=1 pad=1 unless - // noted. Together the first 6 entries account for ~89% of all conv time. - std::vector perf_configs = { - // #1: 21.25% — (1,128,36,48)->(1,128,36,48) - {InputDims(1, 128, 36, 48), - 128, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // #2: 20.68% — (1,256,18,24)->(1,256,18,24) - {InputDims(1, 256, 18, 24), - 256, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // #3: 20.01% — (1,64,72,96)->(1,64,72,96) - {InputDims(1, 64, 72, 96), - 64, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // #4: 13.25% — (1,32,144,192)->(1,32,144,192) - {InputDims(1, 32, 144, 192), - 32, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // #5: 6.74% — (1,64,36,48)->(1,64,36,48) - {InputDims(1, 64, 36, 48), - 64, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // #6: 5.90% — (1,32,72,96)->(1,32,72,96) - {InputDims(1, 32, 72, 96), - 32, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // Secondary cases - // 3x3 stride=2 downsample with channel expansion: 1.52% - {InputDims(1, 32, 72, 96), - 128, - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - // 3x3 stride=1 same-shape, smaller spatial: 1.51% - {InputDims(1, 128, 18, 24), - 128, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // 3x3 stride=1, channel reduction - {InputDims(1, 128, 18, 24), - 64, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 64, 36, 48), - 32, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - // 3x3 stride=2 downsample, same channels - {InputDims(1, 32, 72, 96), - 32, - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 64, 36, 48), - 64, - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - // RGB stem - {InputDims(1, 3, 144, 192), - 32, - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - }; - - // Small shapes used to exercise each im2col intermediate-storage variant - // (buffer / texture2d / texture3d) deterministically and independently of - // the device's auto-selection. All dims < kRefDimSizeLimit so both the FP32 - // and FP16 references validate them. The first two shapes keep M (= H_out * - // W_out) tiny, so the im2col intermediate [1, K_total, H_out, W_out] is small - // and the forced texture3d variant never exercises the large-M texture3d - // Z-layout the auto-selector actually falls back to. The third shape (48x48) - // is included specifically so the forced im2col_tex3d variant exercises the - // texture3d layout (K4 along Z, many spatial rows) at a non-trivial M, while - // still keeping every dim under kRefDimSizeLimit for reference validation. - std::vector per_variant_configs = { - // 3x3 s1 p1, channels multiple of 4 - {InputDims(1, 16, 16, 16), - 16, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - // Non-multiple-of-4 channels exercise the Cin padding path - {InputDims(1, 11, 12, 12), - 13, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - // Larger spatial extent (M = 48*48 = 2304) exercises the texture3d im2col - // layout [1, K_total, H_out, W_out] with K4 along Z at a non-trivial M, - // while all dims stay < kRefDimSizeLimit so both FP32 and FP16 references - // validate it. C_in=16 keeps K = 3*3*16 = 144 (same as the 16x16 case) - // so FP16 accumulation stays within tolerance. - {InputDims(1, 16, 48, 48), - 16, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - }; - - // Boundary pair straddling the should_use_conv2d_im2col() c_out >= 128 - // routing threshold. Spatial dims are tiny (8x8) so the FP32 float reference - // stays cheap, but c_out = 64 / 128 are both >= kRefDimSizeLimit, so these - // get the PERF label. FP32 PERF cases are still numerically VERIFIED (the - // reference's invalid_argument throw that skips the check only fires for - // half), so both implementations are cross-checked against the float - // reference at the boundary. Run all three impls: at c_out = 64 the heuristic - // ("") picks direct on Adreno / im2col on Mali; at c_out = 128 it picks - // im2col on both — and "direct"/"im2col" force each path regardless, proving - // the two implementations agree at the boundary on either device. - std::vector boundary_configs = { - // c_out = 64 (< 128): below the threshold - {InputDims(1, 16, 8, 8), - 64, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - // c_out = 128 (== 128): at/above the threshold - {InputDims(1, 16, 8, 8), - 128, - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - }; - - // Implementation variants exercised for every small ACCU shape: - // "" -> heuristic-routed (should_use_conv2d_im2col picks direct on - // Adreno for small c_out, im2col on Mali) - // "im2col" -> forced im2col/GEMM path - // "direct" -> forced direct sliding-window path (force_direct=true) - // Including "direct" guarantees the direct shader gets reference-checked on - // BOTH devices — without it, Mali would always route "" to im2col and never - // exercise the direct path. - const std::vector impls = {"", "im2col", "direct"}; - // Forced-storage im2col variants for the per-variant ACCU coverage. - const std::vector forced_storage_impls = { - "im2col_buffer", "im2col_tex2d", "im2col_tex3d"}; - - // Generate accuracy test cases for all impls and both dtypes. FP16 small - // shapes get a real reference check (gated in conv2d_reference_impl); we run - // both dtypes so we catch correctness regressions in either path. Large-K - // half stays timing-only via the reference's PERF-shape throw. - const std::vector accu_dtypes = { - vkapi::kFloat, vkapi::kHalf}; - for (const auto& config : accuracy_configs) { - for (auto st : storage_types) { - for (auto dtype : accu_dtypes) { - for (const auto& impl : impls) { - test_cases.push_back( - create_conv2d_test_case(config, dtype, st, layout, impl)); - } - } - } - } - - // Generate per-variant forced-storage ACCU cases (FP32 and FP16) so all - // three im2col intermediate-storage variants get deterministic, - // device-independent, reference-checked coverage at small K. - for (const auto& config : per_variant_configs) { - for (auto st : storage_types) { - for (auto dtype : accu_dtypes) { - for (const auto& impl : forced_storage_impls) { - test_cases.push_back( - create_conv2d_test_case(config, dtype, st, layout, impl)); - } - } - } - } - - // Generate the c_out boundary pair (FP32 only) through all three impls. - // FP32 PERF cases are reference-VERIFIED, so the direct and im2col paths are - // both cross-checked against the float reference at the routing threshold. - for (const auto& config : boundary_configs) { - for (auto st : storage_types) { - for (const auto& impl : impls) { - test_cases.push_back( - create_conv2d_test_case(config, vkapi::kFloat, st, layout, impl)); - } - } - } - - // Generate performance test cases (float and half) for all impls. - for (const auto& config : perf_configs) { - std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; - for (auto dtype : dtypes) { - for (auto st : storage_types) { - for (const auto& impl : impls) { - test_cases.push_back( - create_conv2d_test_case(config, dtype, st, layout, impl)); - } - } - } - } - - return test_cases; -} - -static int64_t conv2d_flop_calculator(const TestCase& test_case) { - auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); - auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); - auto output_sizes = test_case.outputs()[0].get_tensor_sizes(); - - int64_t N = output_sizes[0]; - int64_t C_out = output_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - int64_t C_in = input_sizes[1]; - int64_t K_h = weight_sizes[2]; - int64_t K_w = weight_sizes[3]; - - return 2 * N * C_out * C_in * H_out * W_out * K_h * K_w; -} - -static void reference_impl(TestCase& test_case) { - conv2d_reference_impl(test_case); -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "General Conv2d (SlidingWindow, groups=1) Benchmark" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - execute_test_cases( - generate_conv2d_test_cases, - conv2d_flop_calculator, - "Conv2d", - /*warmup_runs = */ 5, - /*benchmark_runs = */ 20, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/custom_ops/test_conv2d_dw.cpp b/backends/vulkan/test/custom_ops/test_conv2d_dw.cpp deleted file mode 100644 index e0e8a55b815..00000000000 --- a/backends/vulkan/test/custom_ops/test_conv2d_dw.cpp +++ /dev/null @@ -1,439 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include - -#include -#include - -#include "conv2d_utils.h" -#include "utils.h" - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 64; - -struct InputDims { - int64_t N; - int64_t C; - int64_t H; - int64_t W; - - InputDims(int64_t n, int64_t c, int64_t h, int64_t w) - : N(n), C(c), H(h), W(w) {} -}; - -struct Conv2dDwConfig { - InputDims dims; - KernelSize kernel; - Stride stride; - Padding padding; - Dilation dilation; - bool has_bias; -}; - -static int64_t calc_out_size( - int64_t in_size, - int64_t kernel_size, - int64_t stride, - int64_t padding, - int64_t dilation) { - return (in_size + 2 * padding - dilation * (kernel_size - 1) - 1) / stride + - 1; -} - -static TestCase create_conv2d_dw_test_case( - const Conv2dDwConfig& config, - vkapi::ScalarType dtype, - utils::StorageType storage_type, - utils::GPUMemoryLayout memory_layout, - const std::string& impl_selector = "") { - TestCase test_case; - - bool is_perf = config.dims.C > kRefDimSizeLimit || - config.dims.H > kRefDimSizeLimit || config.dims.W > kRefDimSizeLimit; - - std::string prefix = is_perf ? "PERF" : "ACCU"; - std::string storage_str = repr_str(storage_type, memory_layout); - std::string dtype_str = dtype_short(dtype); - std::string bias_str = config.has_bias ? "+bias" : ""; - - int64_t H_out = calc_out_size( - config.dims.H, - config.kernel.h, - config.stride.h, - config.padding.h, - config.dilation.h); - int64_t W_out = calc_out_size( - config.dims.W, - config.kernel.w, - config.stride.w, - config.padding.w, - config.dilation.w); - (void)H_out; - (void)W_out; - - // groups for depthwise conv2d == number of input channels - std::string shape = "[" + std::to_string(config.dims.N) + "," + - std::to_string(config.dims.C) + "," + std::to_string(config.dims.H) + - "," + std::to_string(config.dims.W) + "] k" + - std::to_string(config.kernel.h) + " s" + std::to_string(config.stride.h) + - " p" + std::to_string(config.padding.h) + " d" + - std::to_string(config.dilation.h) + " g" + std::to_string(config.dims.C); - - std::string suffix = bias_str; - if (!impl_selector.empty()) { - if (!suffix.empty()) { - suffix += " "; - } - suffix += "[" + impl_selector + "]"; - } - - std::string name = - make_test_label(prefix, dtype_str, dtype_str, shape, storage_str, suffix); - - test_case.set_name(name); - test_case.set_operator_name("test_etvk.test_conv2d_dw.default"); - - // Input tensor [N, C, H, W] - ValueSpec input( - {config.dims.N, config.dims.C, config.dims.H, config.dims.W}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - - // Weight tensor [C, 1, K_h, K_w] - constant - ValueSpec weight( - {config.dims.C, 1, config.kernel.h, config.kernel.w}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - weight.set_constant(true); - - test_case.add_input_spec(input); - test_case.add_input_spec(weight); - - // Bias (or none) - if (config.has_bias) { - ValueSpec bias( - {config.dims.C}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - bias.set_constant(true); - test_case.add_input_spec(bias); - } else { - ValueSpec none_bias(static_cast(0)); - none_bias.set_none(true); - test_case.add_input_spec(none_bias); - } - - // stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w - test_case.add_input_spec(ValueSpec(static_cast(config.stride.h))); - test_case.add_input_spec(ValueSpec(static_cast(config.stride.w))); - test_case.add_input_spec(ValueSpec(static_cast(config.padding.h))); - test_case.add_input_spec(ValueSpec(static_cast(config.padding.w))); - test_case.add_input_spec(ValueSpec(static_cast(config.dilation.h))); - test_case.add_input_spec(ValueSpec(static_cast(config.dilation.w))); - - // impl_selector string - test_case.add_input_spec(ValueSpec::make_string(impl_selector)); - - // Output tensor [N, C, H_out, W_out] - ValueSpec output( - {config.dims.N, config.dims.C, H_out, W_out}, - dtype, - storage_type, - memory_layout, - DataGenType::ZEROS); - test_case.add_output_spec(output); - - if (dtype == vkapi::kHalf) { - test_case.set_abs_tolerance(1e-1f); - test_case.set_rel_tolerance(1e-2f); - } else { - test_case.set_abs_tolerance(1e-3f); - test_case.set_rel_tolerance(1e-3f); - } - - test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); - - return test_case; -} - -// Reference implementation for depthwise conv2d -static void conv2d_dw_reference_impl(TestCase& test_case) { - const ValueSpec& input = test_case.inputs()[0]; - const ValueSpec& weight = test_case.inputs()[1]; - const ValueSpec& bias_spec = test_case.inputs()[2]; - ValueSpec& output = test_case.outputs()[0]; - - if (input.dtype != vkapi::kFloat) { - throw std::invalid_argument("Reference only supports float"); - } - - auto input_sizes = input.get_tensor_sizes(); - auto weight_sizes = weight.get_tensor_sizes(); - auto output_sizes = output.get_tensor_sizes(); - - int64_t N = input_sizes[0]; - int64_t C = input_sizes[1]; - int64_t H_in = input_sizes[2]; - int64_t W_in = input_sizes[3]; - int64_t K_h = weight_sizes[2]; - int64_t K_w = weight_sizes[3]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - int64_t stride_h = test_case.inputs()[3].get_int_value(); - int64_t stride_w = test_case.inputs()[4].get_int_value(); - int64_t padding_h = test_case.inputs()[5].get_int_value(); - int64_t padding_w = test_case.inputs()[6].get_int_value(); - int64_t dilation_h = test_case.inputs()[7].get_int_value(); - int64_t dilation_w = test_case.inputs()[8].get_int_value(); - - auto& input_data = input.get_float_data(); - auto& weight_data = weight.get_float_data(); - auto& ref_data = output.get_ref_float_data(); - ref_data.resize(N * C * H_out * W_out, 0.0f); - - for (int64_t n = 0; n < N; ++n) { - for (int64_t c = 0; c < C; ++c) { - for (int64_t oh = 0; oh < H_out; ++oh) { - for (int64_t ow = 0; ow < W_out; ++ow) { - float sum = 0.0f; - for (int64_t kh = 0; kh < K_h; ++kh) { - for (int64_t kw = 0; kw < K_w; ++kw) { - int64_t ih = oh * stride_h - padding_h + kh * dilation_h; - int64_t iw = ow * stride_w - padding_w + kw * dilation_w; - if (ih >= 0 && ih < H_in && iw >= 0 && iw < W_in) { - float in_val = input_data - [n * (C * H_in * W_in) + c * (H_in * W_in) + ih * W_in + - iw]; - // weight is [C, 1, K_h, K_w] - float w_val = weight_data[c * (K_h * K_w) + kh * K_w + kw]; - sum += in_val * w_val; - } - } - } - if (!bias_spec.is_none()) { - auto& bias_data = bias_spec.get_float_data(); - sum += bias_data[c]; - } - ref_data - [n * (C * H_out * W_out) + c * (H_out * W_out) + oh * W_out + - ow] = sum; - } - } - } - } -} - -static std::vector generate_conv2d_dw_test_cases() { - std::vector test_cases; - - std::vector storage_types = {utils::kTexture3D}; - utils::GPUMemoryLayout layout = utils::kChannelsPacked; - - // Accuracy shapes (small enough for float reference validation) - std::vector accuracy_configs = { - {InputDims(1, 8, 16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 8, 16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - true}, - {InputDims(1, 8, 16, 16), - KernelSize(5, 5), - Stride(1, 1), - Padding(2, 2), - Dilation(1, 1), - false}, - {InputDims(1, 8, 16, 16), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - // Non-multiple-of-4 channels - {InputDims(1, 11, 16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 3, 16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - }; - - // EdgeTAM depthwise shapes (from profiling data) - std::vector perf_configs = { - // Backbone stem and early stages - {InputDims(1, 24, 512, 512), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 48, 256, 256), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 48, 256, 256), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 96, 128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 96, 128, 128), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 192, 64, 64), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 192, 64, 64), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 384, 32, 32), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - // 5x5 kernels - {InputDims(1, 48, 256, 256), - KernelSize(5, 5), - Stride(1, 1), - Padding(2, 2), - Dilation(1, 1), - false}, - {InputDims(1, 96, 128, 128), - KernelSize(5, 5), - Stride(1, 1), - Padding(2, 2), - Dilation(1, 1), - false}, - // FPN/Neck - {InputDims(1, 256, 256, 256), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - {InputDims(1, 256, 128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - false}, - }; - - // Generate accuracy test cases (float only) - for (const auto& config : accuracy_configs) { - for (auto st : storage_types) { - test_cases.push_back( - create_conv2d_dw_test_case(config, vkapi::kFloat, st, layout)); - } - } - - // Generate performance test cases (float and half) - for (const auto& config : perf_configs) { - std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; - for (auto dtype : dtypes) { - for (auto st : storage_types) { - // Auto-selection (empty impl_selector) - test_cases.push_back( - create_conv2d_dw_test_case(config, dtype, st, layout)); - - // Force b4x2 variant - test_cases.push_back( - create_conv2d_dw_test_case(config, dtype, st, layout, "b4x2")); - - // Force b1x1 variant (only for 3x3 kernels; for 5x5 it falls back - // to default, but we still generate it to test the fallback path) - test_cases.push_back( - create_conv2d_dw_test_case(config, dtype, st, layout, "b1x1")); - } - } - } - - return test_cases; -} - -static int64_t conv2d_dw_flop_calculator(const TestCase& test_case) { - auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); - auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); - auto output_sizes = test_case.outputs()[0].get_tensor_sizes(); - - int64_t N = output_sizes[0]; - int64_t C = output_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - int64_t K_h = weight_sizes[2]; - int64_t K_w = weight_sizes[3]; - - // Each output element: K_h * K_w multiplies + (K_h * K_w - 1) adds - return 2 * N * C * H_out * W_out * K_h * K_w; -} - -static void reference_impl(TestCase& test_case) { - conv2d_dw_reference_impl(test_case); -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Depthwise Conv2d Benchmark" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_conv2d_dw_test_cases, - conv2d_dw_flop_calculator, - "Conv2dDW", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/custom_ops/test_conv2d_pw.cpp b/backends/vulkan/test/custom_ops/test_conv2d_pw.cpp deleted file mode 100644 index 30cdb352248..00000000000 --- a/backends/vulkan/test/custom_ops/test_conv2d_pw.cpp +++ /dev/null @@ -1,267 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include - -#include -#include - -#include "utils.h" - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 64; - -struct Conv2dPwConfig { - int64_t N; - int64_t C_in; - int64_t C_out; - int64_t H; - int64_t W; - bool has_bias; -}; - -static TestCase create_conv2d_pw_test_case( - const Conv2dPwConfig& config, - vkapi::ScalarType dtype, - utils::StorageType storage_type, - utils::GPUMemoryLayout memory_layout) { - TestCase test_case; - - bool is_perf = config.C_in > kRefDimSizeLimit || - config.C_out > kRefDimSizeLimit || config.H > kRefDimSizeLimit || - config.W > kRefDimSizeLimit; - - std::string prefix = is_perf ? "PERF" : "ACCU"; - std::string storage_str = repr_str(storage_type, memory_layout); - std::string dtype_str = dtype_short(dtype); - std::string bias_str = config.has_bias ? "+bias" : ""; - - // Pointwise conv2d: kernel 1x1, stride 1, pad 0, dilation 1, groups 1 - std::string shape = "[" + std::to_string(config.N) + "," + - std::to_string(config.C_in) + "," + std::to_string(config.H) + "," + - std::to_string(config.W) + "]x[" + std::to_string(config.C_out) + "," + - std::to_string(config.C_in) + ",1,1] s1 p0 d1 g1"; - - std::string name = make_test_label( - prefix, dtype_str, dtype_str, shape, storage_str, bias_str); - - test_case.set_name(name); - test_case.set_operator_name("test_etvk.test_conv2d_pw.default"); - - // Input tensor [N, C_in, H, W] - ValueSpec input( - {config.N, config.C_in, config.H, config.W}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - - // Weight tensor [C_out, C_in, 1, 1] - constant - ValueSpec weight( - {config.C_out, config.C_in, 1, 1}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - weight.set_constant(true); - - test_case.add_input_spec(input); - test_case.add_input_spec(weight); - - // Bias (or none) - if (config.has_bias) { - ValueSpec bias( - {config.C_out}, - dtype, - storage_type, - memory_layout, - DataGenType::RANDOM); - bias.set_constant(true); - test_case.add_input_spec(bias); - } else { - ValueSpec none_bias(static_cast(0)); - none_bias.set_none(true); - test_case.add_input_spec(none_bias); - } - - // impl_selector - ValueSpec impl_selector_spec = ValueSpec::make_string("default"); - test_case.add_input_spec(impl_selector_spec); - - // Output tensor [N, C_out, H, W] - ValueSpec output( - {config.N, config.C_out, config.H, config.W}, - dtype, - storage_type, - memory_layout, - DataGenType::ZEROS); - test_case.add_output_spec(output); - - if (dtype == vkapi::kHalf) { - test_case.set_abs_tolerance(1e-1f); - test_case.set_rel_tolerance(1e-2f); - } else { - test_case.set_abs_tolerance(1e-3f); - test_case.set_rel_tolerance(1e-3f); - } - - test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); - - return test_case; -} - -// Reference implementation: pointwise conv2d is essentially a matmul -// output[n][c_out][h][w] = bias[c_out] + -// sum_over_c_in(input[n][c_in][h][w] * weight[c_out][c_in][0][0]) -static void conv2d_pw_reference_impl(TestCase& test_case) { - // input[0], weight[1], bias[2], impl_selector[3] - const ValueSpec& input = test_case.inputs()[0]; - const ValueSpec& weight = test_case.inputs()[1]; - const ValueSpec& bias_spec = test_case.inputs()[2]; - ValueSpec& output = test_case.outputs()[0]; - - if (input.dtype != vkapi::kFloat) { - throw std::invalid_argument("Reference only supports float"); - } - - auto input_sizes = input.get_tensor_sizes(); - auto weight_sizes = weight.get_tensor_sizes(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t H = input_sizes[2]; - int64_t W = input_sizes[3]; - int64_t C_out = weight_sizes[0]; - - auto& input_data = input.get_float_data(); - auto& weight_data = weight.get_float_data(); - auto& ref_data = output.get_ref_float_data(); - ref_data.resize(N * C_out * H * W, 0.0f); - - for (int64_t n = 0; n < N; ++n) { - for (int64_t co = 0; co < C_out; ++co) { - for (int64_t h = 0; h < H; ++h) { - for (int64_t w = 0; w < W; ++w) { - float sum = 0.0f; - for (int64_t ci = 0; ci < C_in; ++ci) { - float in_val = - input_data[n * (C_in * H * W) + ci * (H * W) + h * W + w]; - // weight is [C_out, C_in, 1, 1] - float w_val = weight_data[co * C_in + ci]; - sum += in_val * w_val; - } - if (!bias_spec.is_none()) { - auto& bias_data = bias_spec.get_float_data(); - sum += bias_data[co]; - } - ref_data[n * (C_out * H * W) + co * (H * W) + h * W + w] = sum; - } - } - } - } -} - -static std::vector generate_conv2d_pw_test_cases() { - std::vector test_cases; - - // Conv2d shaders are texture-only and require channels-packed layout - std::vector storage_types = {utils::kTexture3D}; - utils::GPUMemoryLayout layout = utils::kChannelsPacked; - - // Accuracy shapes (small enough for float reference validation) - std::vector accuracy_configs = { - {1, 16, 32, 8, 8, false}, - {1, 32, 16, 8, 8, false}, - {1, 16, 32, 8, 8, true}, - {1, 48, 96, 16, 16, false}, - {1, 96, 48, 16, 16, false}, - // Non-multiple-of-4 channels - {1, 13, 27, 8, 8, false}, - {1, 33, 17, 8, 8, false}, - }; - - // EdgeTAM performance shapes - std::vector perf_configs = { - // EdgeTAM backbone stages - {1, 48, 96, 256, 256, false}, - {1, 96, 48, 256, 256, false}, - {1, 96, 192, 128, 128, false}, - {1, 192, 96, 128, 128, false}, - {1, 192, 384, 64, 64, false}, - {1, 384, 192, 64, 64, false}, - {1, 384, 768, 32, 32, false}, - {1, 768, 384, 32, 32, false}, - // EdgeTAM FPN/Neck - {1, 48, 256, 256, 256, false}, - {1, 256, 32, 256, 256, false}, - {1, 96, 256, 128, 128, false}, - {1, 256, 64, 128, 128, false}, - }; - - // Generate accuracy test cases (float only) - for (const auto& config : accuracy_configs) { - for (auto st : storage_types) { - test_cases.push_back( - create_conv2d_pw_test_case(config, vkapi::kFloat, st, layout)); - } - } - - // Generate performance test cases (float and half) - for (const auto& config : perf_configs) { - std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; - for (auto dtype : dtypes) { - for (auto st : storage_types) { - test_cases.push_back( - create_conv2d_pw_test_case(config, dtype, st, layout)); - } - } - } - - return test_cases; -} - -static int64_t conv2d_pw_flop_calculator(const TestCase& test_case) { - auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); - auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t H = input_sizes[2]; - int64_t W = input_sizes[3]; - int64_t C_out = weight_sizes[0]; - - return 2 * N * C_out * C_in * H * W; -} - -static void reference_impl(TestCase& test_case) { - conv2d_pw_reference_impl(test_case); -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Pointwise Conv2d (1x1) Benchmark" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_conv2d_pw_test_cases, - conv2d_pw_flop_calculator, - "Conv2dPW", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/custom_ops/test_coopmat_linear_bench.cpp b/backends/vulkan/test/custom_ops/test_coopmat_linear_bench.cpp deleted file mode 100644 index d57d41266a5..00000000000 --- a/backends/vulkan/test/custom_ops/test_coopmat_linear_bench.cpp +++ /dev/null @@ -1,406 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -// Consolidated coopmat-vs-tiled microbenchmark for the two int4-quantized -// linear types at Llama 3.1 8B prefill shapes: -// 4w = linear_q4gsw (weight-only int4) -// 8da4w = linear_dq8ca_q4gsw (dyn-act int8 x int4 weight) -// -// Baseline (tiled) is selected by Texture3D+Half output storage; coopmat is -// selected by Buffer+Half (the runtime gate in QuantizedLinear.cpp picks the -// _coopmat shader when M%64==0, N%64==0, K%32==0, subgroup==64). Perf-only: -// no CPU reference is run (correctness is covered by the per-op test_*_linear -// benches at small shapes). - -#include -#include -#include -#include -#include -#include -#include -#include "utils.h" - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -struct LinearConfig { - int64_t M; - int64_t K; - int64_t N; - int64_t group_size; // only meaningful for 4-bit - std::string op_name; -}; - -static bool is_dq8ca(const std::string& op) { - return op.find("dq8ca") != std::string::npos; -} -static bool is_4bit(const std::string& op) { - return op.find("q4gsw") != std::string::npos; -} - -// Build one test case for the given op at (storage, half dtype), no bias. -static TestCase make_case(const LinearConfig& cfg, utils::StorageType storage) { - const vkapi::ScalarType dt = vkapi::kHalf; - TestCase tc; - const std::string storage_str = - (storage == utils::kTexture3D) ? "Texture3D" : "Buffer"; - tc.set_name( - cfg.op_name + "_M" + std::to_string(cfg.M) + "_K" + - std::to_string(cfg.K) + "_N" + std::to_string(cfg.N) + "_" + storage_str); - tc.set_operator_name("et_vk." + cfg.op_name + ".default"); - - ValueSpec input( - {cfg.M, cfg.K}, dt, storage, utils::kWidthPacked, DataGenType::RANDINT); - - // dynamic per-row activation scale/zp (dq8ca only) - ValueSpec input_scale( - {1, cfg.M}, dt, storage, utils::kWidthPacked, DataGenType::RANDOM_SCALES); - input_scale.set_constant(true); - ValueSpec input_zp( - {1, cfg.M}, - vkapi::kChar, - storage, - utils::kWidthPacked, - DataGenType::RANDINT); - input_zp.set_constant(true); - - // weight + scales + sums depend on 4-bit vs 8-bit - const bool four = is_4bit(cfg.op_name); - ValueSpec qweight( - four ? std::vector{cfg.N, cfg.K / 2} - : std::vector{cfg.N, cfg.K}, - four ? vkapi::kByte : vkapi::kChar, - storage, - utils::kWidthPacked, - four ? DataGenType::RANDINT4 : DataGenType::RANDINT8); - qweight.set_constant(true); - if (four) { - qweight.set_int4(true); - } - - std::vector scales_size = four - ? std::vector{cfg.K / cfg.group_size, cfg.N} - : std::vector{cfg.N}; - ValueSpec weight_scales( - scales_size, - dt, - storage, - utils::kWidthPacked, - DataGenType::RANDOM_SCALES); - weight_scales.set_constant(true); - - ValueSpec weight_sums( - scales_size, - vkapi::kInt, - storage, - utils::kWidthPacked, - DataGenType::ZEROS); - weight_sums.set_constant(true); - if (four) { - compute_weight_sums_4bit_grouped( - weight_sums, qweight, cfg.K / cfg.group_size, cfg.N, cfg.group_size); - } else { - compute_weight_sums(weight_sums, qweight, cfg.N, cfg.K); - } - - ValueSpec group_size_spec(static_cast(cfg.group_size)); - - ValueSpec bias({cfg.N}, dt, storage, utils::kWidthPacked, DataGenType::ZEROS); - bias.set_constant(true); - bias.set_none(true); - - ValueSpec output( - {cfg.M, cfg.N}, dt, storage, utils::kWidthPacked, DataGenType::ZEROS); - - // assemble per op signature - if (cfg.op_name == "linear_q4gsw") { - tc.add_input_spec(input); - tc.add_input_spec(qweight); - tc.add_input_spec(weight_scales); - tc.add_input_spec(group_size_spec); - tc.add_input_spec(bias); - } else if (cfg.op_name == "linear_dq8ca_q4gsw") { - tc.add_input_spec(input); - tc.add_input_spec(input_scale); - tc.add_input_spec(input_zp); - tc.add_input_spec(qweight); - tc.add_input_spec(weight_sums); - tc.add_input_spec(weight_scales); - tc.add_input_spec(group_size_spec); - tc.add_input_spec(bias); - } - tc.add_output_spec(output); - return tc; -} - -// ---- correctness reference for all four ops; oversized shapes (the perf -// cases) throw -> framework marks them SKIPPED. For dq8ca the activation -// quant round-trip (round(x/scale)+zp) is mirrored in fp32; this is exact -// (not just close) for the correctness data below, which uses scale=1/16, -// zp=0 and activations that are multiples of 1/16, so fp16-vs-fp32 -// divergence cannot occur. ---- -static std::vector as_f(const ValueSpec& s) { - if (s.dtype == vkapi::kFloat) { - return s.get_float_data(); - } - const auto& h = s.get_half_data(); - std::vector o(h.size()); - for (size_t i = 0; i < h.size(); ++i) { - o[i] = half_to_float(h[i]); - } - return o; -} -static void bench_reference(TestCase& tc) { - const std::string op = tc.operator_name(); - const bool dq8ca = op.find("dq8ca") != std::string::npos; - const bool four = op.find("q4gsw") != std::string::npos; - const ValueSpec& in = tc.inputs()[0]; - ValueSpec& out = tc.outputs()[0]; - const auto is = in.get_tensor_sizes(); - const int64_t M = is[0], K = is[1]; - const int64_t N = out.get_tensor_sizes()[1]; - if (M > 256 || K > 256 || N > 256) { - throw std::invalid_argument("ref: too big"); - } - // input layouts: weight-only = {in, w, w_scales, [group], bias}; - // dq8ca = {in, in_scale, in_zp, w, w_sums, w_scales, [group], bias} - const ValueSpec& w = tc.inputs()[dq8ca ? 3 : 1]; - const ValueSpec& sc = tc.inputs()[dq8ca ? 5 : 2]; - const int64_t group = four ? tc.inputs()[dq8ca ? 6 : 3].get_int_value() : K; - const ValueSpec& bias = tc.inputs()[dq8ca ? (four ? 7 : 6) : (four ? 4 : 3)]; - const bool has_bias = !bias.is_none(); - - const std::vector inf = as_f(in); - const std::vector scf = as_f(sc); - const std::vector bf = has_bias ? as_f(bias) : std::vector(); - const std::vector in_scale = - dq8ca ? as_f(tc.inputs()[1]) : std::vector(); - const std::vector& in_zp = - dq8ca ? tc.inputs()[2].get_int8_data() : std::vector(); - const std::vector& w4 = - four ? w.get_uint8_data() : std::vector(); // [N, K/2] nibbles - const std::vector& w8 = - four ? std::vector() : w.get_int8_data(); // [N, K] - - auto& ref = out.get_ref_float_data(); - ref.resize(M * N); - for (int64_t m = 0; m < M; ++m) { - const float s_in = dq8ca ? in_scale[m] : 1.0f; - const int zp = dq8ca ? int(in_zp[m]) : 0; - for (int64_t n = 0; n < N; ++n) { - float acc = 0.0f; - for (int64_t k = 0; k < K; ++k) { - float a = inf[m * K + k]; - if (dq8ca) { - float q = std::round(a / s_in) + float(zp); - q = std::min(std::max(q, -128.0f), 127.0f); - a = q - float(zp); - } - int wv; - if (four) { - const uint8_t byte = w4[n * (K / 2) + k / 2]; - const int nib = (k & 1) ? ((byte >> 4) & 0xF) : (byte & 0xF); - wv = nib - 8; - } else { - wv = w8[n * K + k]; - } - const float w_scale = four ? scf[(k / group) * N + n] : scf[n]; - acc += a * float(wv) * w_scale; - } - float r = dq8ca ? acc * s_in : acc; - if (has_bias) { - r += bf[n]; - } - ref[m * N + n] = r; - } - } -} - -// Llama 3.1 8B linear weight shapes (K,N) at prefill M (multiple of 64 so -// coopmat fires). -static const std::vector> kShapes = { - {4096, 4096}, // q_proj / o_proj - {4096, 1024}, // k_proj / v_proj (GQA) - {4096, 14336}, // gate_proj / up_proj - {14336, 4096}, // down_proj -}; -static const std::vector kOps = { - "linear_q4gsw", - "linear_dq8ca_q4gsw"}; -static constexpr int64_t kM = 1024; -static constexpr int64_t kGroup = 128; - -// Generation order: for each op, for each shape -> {Texture3D, Buffer}. -// Summary pairs results [2i]=tiled, [2i+1]=coopmat. -std::vector generate_cases() { - std::vector cases; - // COOPMAT_BENCH_CORRECTNESS_ONLY=1 skips the (slow) M=1024 perf cases and - // runs just the small correctness matrix below. - const bool correctness_only = - std::getenv("COOPMAT_BENCH_CORRECTNESS_ONLY") != nullptr; - if (!correctness_only) { - for (const auto& op : kOps) { - for (const auto& kn : kShapes) { - LinearConfig cfg{kM, kn.first, kn.second, kGroup, op}; - cases.push_back(make_case(cfg, utils::kTexture3D)); // tiled baseline - cases.push_back( - make_case(cfg, utils::kBuffer)); // coopmat (gate-permitting) - } - } - } - // Correctness: small aligned {64,128,64} cases for ALL FOUR ops; the buffer - // case fires the coopmat shader, validated by bench_reference (the perf - // cases above are skipped by it). POSITIVE well-conditioned data (no fp16 - // cancellation): activations are multiples of 1/16 in [0.5,1.375]; int4 - // nibbles in {9..14} (-> weight +1..+6) / int8 weights in {1..6}. For dq8ca - // the per-row activation scale is forced to 1/16 with zp=0 so the dynamic - // int8 quant round-trip is EXACT in both fp16 and fp32 (quantized values - // 8..22) and the fp32 reference is valid. fp16~=fp32 throughout, so a tight - // tolerance validates shader structure (catches zero-subtile bugs) while - // ignoring benign fp16 noise. Texture3D = tiled, Buffer = coopmat. - // Shapes align to BOTH coopmat geometries (64x64x32 legacy, 128x128x16 - // double-buffered); the second shape dispatches a multi-workgroup grid for - // both, covering the gl_WorkGroupID-derived tile offsets in the store - // address math. - static const std::vector kCorrectnessShapes = { - {64, 128, 64, 64, ""}, - {128, 256, 128, 64, ""}, - {128, 128, 128, 64, ""}, - {256, 256, 256, 64, ""}, - // Discriminators for the tiled-texture cube-shape failure: - {128, 128, 256, 64, ""}, // M == K only - {256, 128, 128, 64, ""}, // K == N only - {64, 128, 256, 64, ""}, // K > M, K < N - {256, 128, 64, 64, ""}}; // K < M, K > N - for (const auto& op : kOps) { - for (const auto& shape : kCorrectnessShapes) { - LinearConfig cfg{shape.M, shape.K, shape.N, shape.group_size, op}; - const bool dq = is_dq8ca(op); - const bool four = is_4bit(op); - for (auto st : {utils::kTexture3D, utils::kBuffer}) { - TestCase t = make_case(cfg, st); - auto& hin = t.inputs()[0].get_half_data(); - for (size_t i = 0; i < hin.size(); ++i) { - hin[i] = float_to_half(0.5f + 0.125f * float(i % 8)); - } - const size_t w_idx = dq ? 3 : 1; - if (four) { - auto& wq = t.inputs()[w_idx].get_uint8_data(); - const uint8_t kPos[6] = {0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE}; - for (size_t i = 0; i < wq.size(); ++i) { - wq[i] = kPos[i % 6]; - } - } else { - auto& wq = t.inputs()[w_idx].get_int8_data(); - for (size_t i = 0; i < wq.size(); ++i) { - wq[i] = int8_t(1 + (i % 6)); - } - } - if (dq) { - auto& hs = t.inputs()[1].get_half_data(); - std::fill(hs.begin(), hs.end(), float_to_half(0.0625f)); - auto& zp = t.inputs()[2].get_int8_data(); - std::fill(zp.begin(), zp.end(), int8_t(0)); - // weights were overwritten above -> recompute the sums - if (four) { - compute_weight_sums_4bit_grouped( - t.inputs()[4], - t.inputs()[w_idx], - cfg.K / cfg.group_size, - cfg.N, - cfg.group_size); - } else { - compute_weight_sums(t.inputs()[4], t.inputs()[w_idx], cfg.N, cfg.K); - } - } - t.set_abs_tolerance(0.5f); - t.set_rel_tolerance(0.05f); - cases.push_back(t); - } - } - } - return cases; -} - -int64_t flop_calc(const TestCase& tc) { - const auto& in = tc.inputs()[0].get_tensor_sizes(); - const auto& out = tc.outputs()[0].get_tensor_sizes(); - const int64_t M = in[0], K = in[1], N = out[1]; - return 2 * M * N * K; // MAC = 2 flops -} - -int main() { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "Coopmat vs Tiled quantized-linear microbench (Llama 3.1 8B shapes, M=" - << kM << ")" << std::endl; - print_separator(); - - auto results = execute_test_cases( - generate_cases, - flop_calc, - "CoopmatLinearBench", - /*warmup=*/3, - /*runs=*/5, - /*reference=*/bench_reference); - - // Summary table: pair tiled (even idx) vs coopmat (odd idx) per (op, shape). - // GFLOP/s computed from avg GPU time and 2*M*N*K flops. - if (results.size() < kOps.size() * kShapes.size() * 2) { - return 0; // correctness-only run: no perf cases to summarize - } - auto gflops = [](float time_us, int64_t M, int64_t K, int64_t N) -> float { - return time_us > 0 ? (2.0f * M * N * K) / (time_us * 1e3f) : 0.0f; - }; - // The result's kernel_name is the test-case name; the dispatched shader - // names are in the per-shader timings (dq8ca cases also run a - // quantize_and_pack shader, so pick the linear_* one). - auto linear_kernel = [](const BenchmarkResult& r) -> std::string { - std::string name = r.get_kernel_name(); - for (const auto& st : r.get_shader_timings()) { - if (st.shader_name.find("linear_") != std::string::npos) { - name = st.shader_name; - } - } - return name; - }; - std::cout - << "\n================ SUMMARY: tiled vs coopmat (GFLOP/s) ================\n"; - std::cout << std::left << std::setw(22) << "op" << std::setw(13) - << "shape(K,N)" << std::right << std::setw(10) << "tiled" - << std::setw(10) << "coopmat" << std::setw(9) << "speedup" - << " coopmat kernel\n"; - size_t idx = 0; - for (const auto& op : kOps) { - for (const auto& kn : kShapes) { - const float t_us = results[idx].get_avg_time_us(); - const float c_us = results[idx + 1].get_avg_time_us(); - const std::string coop_kernel = linear_kernel(results[idx + 1]); - const float tiled = gflops(t_us, kM, kn.first, kn.second); - const float coop = gflops(c_us, kM, kn.first, kn.second); - idx += 2; - // If the "coopmat" (buffer) case did not actually pick a _coopmat shader, - // flag it (e.g. shape not gate-eligible). - const bool fired = coop_kernel.find("coopmat") != std::string::npos; - std::cout << std::left << std::setw(22) << op << std::setw(13) - << ("(" + std::to_string(kn.first) + "," + - std::to_string(kn.second) + ")") - << std::right << std::setw(10) << std::fixed - << std::setprecision(1) << tiled << std::setw(10) << coop - << std::setw(8) << std::setprecision(2) - << (tiled > 0 ? coop / tiled : 0.0f) << "x" - << (fired ? " " : " !") << coop_kernel << "\n"; - } - } - std::cout << "(! = buffer case did NOT dispatch a coopmat shader)\n"; - return 0; -} diff --git a/backends/vulkan/test/custom_ops/test_embedding_q4gsw.cpp b/backends/vulkan/test/custom_ops/test_embedding_q4gsw.cpp deleted file mode 100644 index 7f38b4de77b..00000000000 --- a/backends/vulkan/test/custom_ops/test_embedding_q4gsw.cpp +++ /dev/null @@ -1,548 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include - -#include -#include - -#include "utils.h" - -using namespace executorch::vulkan::prototyping; - -using namespace vkcompute; - -struct EmbeddingConfig { - int64_t vocab_size; - int64_t embed_dim; - int64_t group_size; - std::vector indices_shape; - std::string test_case_name = "placeholder"; - vkapi::ScalarType dtype = vkapi::kHalf; - vkapi::ScalarType scales_dtype = vkapi::kHalf; - utils::StorageType storage_type = utils::kBuffer; - bool is_linear_weight = false; -}; - -// CPU reference: unpack 4-bit weights, dequantize, and perform embedding lookup -void embedding_4bit_reference(TestCase& tc) { - auto& weight_spec = tc.inputs()[0]; - auto& scales_spec = tc.inputs()[1]; - int32_t group_size = tc.inputs()[2].get_int_value(); - auto& indices_spec = tc.inputs()[3]; - bool is_linear_weight = tc.inputs()[4].get_bool_value(); - auto& output_spec = tc.outputs()[0]; - - weight_spec.ensure_data_generated(); - scales_spec.ensure_data_generated(); - indices_spec.ensure_data_generated(); - - const auto& weight_data = weight_spec.get_uint8_data(); - const auto& indices_data = indices_spec.get_int32_data(); - - bool scales_are_half = (scales_spec.dtype == vkapi::kHalf); - - int64_t packed_dim = weight_spec.sizes[1]; - int64_t embed_dim = packed_dim * 2; - int64_t groups_per_row = scales_spec.sizes[1]; - - int64_t num_indices = 1; - for (auto s : indices_spec.sizes) { - num_indices *= s; - } - - int64_t total_output = num_indices * embed_dim; - - // Always populate ref_float_data so the caching framework can distribute it - output_spec.get_ref_float_data().resize(total_output); - - bool output_is_half = (output_spec.dtype == vkapi::kHalf); - if (output_is_half) { - output_spec.get_ref_half_data().resize(total_output); - } - - for (int64_t i = 0; i < num_indices; ++i) { - int32_t idx = indices_data[i]; - for (int64_t d = 0; d < embed_dim; ++d) { - int64_t packed_idx = d / 2; - uint8_t packed_byte = weight_data[idx * packed_dim + packed_idx]; - - // Unpack: packed_byte = (even_val + 8) << 4 | (odd_val + 8) - // Even d -> high nibble, odd d -> low nibble - // For linear weight packing, nibble order is swapped - int int4_val; - if (d % 2 == 0) { - if (is_linear_weight) { - int4_val = static_cast(packed_byte & 0xF) - 8; - } else { - int4_val = static_cast(packed_byte >> 4) - 8; - } - } else { - if (is_linear_weight) { - int4_val = static_cast(packed_byte >> 4) - 8; - } else { - int4_val = static_cast(packed_byte & 0xF) - 8; - } - } - - int64_t group_idx = d / group_size; - int64_t scale_idx = idx * groups_per_row + group_idx; - - float scale; - if (scales_are_half) { - uint16_t scale_half = scales_spec.get_half_data()[scale_idx]; - scale = half_to_float(scale_half); - } else { - scale = scales_spec.get_float_data()[scale_idx]; - } - - float result = static_cast(int4_val) * scale; - - // Always store float reference - output_spec.get_ref_float_data()[i * embed_dim + d] = result; - - if (output_is_half) { - output_spec.get_ref_half_data()[i * embed_dim + d] = - float_to_half(result); - } - } - } -} - -TestCase create_test_case(const EmbeddingConfig& config) { - TestCase test_case; - - // Compute the output shape for label: indices_shape + [embed_dim] - std::vector output_shape_for_label = config.indices_shape; - output_shape_for_label.push_back(config.embed_dim); - - // Treat any case that uses the large llama vocab/embed sizes as PERF. - // Otherwise default to ACCU (no PERF distinction is exercised by this op). - bool is_perf = config.vocab_size > 1024 || config.embed_dim > 1024; - std::string prefix = is_perf ? "PERF" : "ACCU"; - - std::string in_dtype = dtype_short(vkapi::kInt); // indices are i32 - std::string out_dtype = dtype_short(config.dtype); - - std::string storage_str = repr_str(config.storage_type, utils::kWidthPacked); - - std::string shape_str = shape_bracket(config.indices_shape) + "x[" + - std::to_string(config.vocab_size) + "," + - std::to_string(config.embed_dim) + "]"; - shape_str += " g" + std::to_string(config.group_size); - if (config.is_linear_weight) { - shape_str += " lw"; - } - std::string suffix; - if (config.scales_dtype == vkapi::kFloat) { - suffix = "[f32_scales]"; - } - std::string name = make_test_label( - prefix, in_dtype, out_dtype, shape_str, storage_str, suffix); - test_case.set_name(name); - test_case.set_operator_name("et_vk.embedding_q4gsw.default"); - test_case.set_shader_filter({}); - - // Weight: [vocab_size, embed_dim / 2] packed uint8 - ValueSpec weight( - {config.vocab_size, config.embed_dim / 2}, - vkapi::kByte, - utils::kBuffer, - utils::kWidthPacked, - DataGenType::RANDINT4); - weight.set_constant(true); - test_case.add_input_spec(weight); - - // Weight scales: [vocab_size, groups_per_row] - int64_t groups_per_row = config.embed_dim / config.group_size; - ValueSpec weight_scales( - {config.vocab_size, groups_per_row}, - config.scales_dtype, - utils::kBuffer, - utils::kWidthPacked, - DataGenType::RANDOM_SCALES); - weight_scales.set_constant(true); - test_case.add_input_spec(weight_scales); - - // Group size: int scalar - ValueSpec group_size_spec(static_cast(config.group_size)); - test_case.add_input_spec(group_size_spec); - - // Indices: [batch, seq_len] int32 - ValueSpec indices( - config.indices_shape, - vkapi::kInt, - config.storage_type, - utils::kWidthPacked, - DataGenType::RANDINT); - - // Clamp indices to valid vocab range - indices.ensure_data_generated(); - for (auto& idx : indices.get_int32_data()) { - idx = std::abs(idx) % config.vocab_size; - } - - test_case.add_input_spec(indices); - - // is_linear_weight: bool scalar - ValueSpec is_linear_weight_spec(config.is_linear_weight); - test_case.add_input_spec(is_linear_weight_spec); - - // Output: indices.shape + [embed_dim] - std::vector output_shape = config.indices_shape; - output_shape.push_back(config.embed_dim); - ValueSpec output( - output_shape, config.dtype, config.storage_type, utils::kWidthPacked); - test_case.add_output_spec(output); - - return test_case; -} - -std::vector generate_test_cases() { - std::vector test_cases; - - // --- is_linear_weight = true --- - - // Basic test with linear weight packing - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_linear_weight", - .is_linear_weight = true})); - - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d_linear_weight", - .is_linear_weight = true})); - - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup_linear_weight", - .is_linear_weight = true})); - - // fp32 output with linear weight - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_fp32_linear_weight", - .dtype = vkapi::kFloat, - .is_linear_weight = true})); - - // Texture3D with linear weight - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_texture_linear_weight", - .storage_type = utils::kTexture3D, - .is_linear_weight = true})); - - // Llama 3.2 1B with linear weight packing - test_cases.push_back(create_test_case( - {.vocab_size = 128256, - .embed_dim = 2048, - .group_size = 32, - .indices_shape = {1, 2047}, - .test_case_name = "llama_3_2_1b_prefill_fp32_linear_weight", - .dtype = vkapi::kFloat, - .storage_type = utils::kBuffer, - .is_linear_weight = true})); - - // --- Half scales (default) --- - - // Basic test: small vocab, small embed_dim - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d"})); - - // 2D indices - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d"})); - - // Larger vocab, multiple groups - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup"})); - - // group_size == embed_dim (single group) - test_cases.push_back(create_test_case( - {.vocab_size = 50, - .embed_dim = 64, - .group_size = 64, - .indices_shape = {2, 4}, - .test_case_name = "single_group"})); - - // fp32 output variants (half scales) - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_fp32", - .dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d_fp32", - .dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup_fp32", - .dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 50, - .embed_dim = 64, - .group_size = 64, - .indices_shape = {2, 4}, - .test_case_name = "single_group_fp32", - .dtype = vkapi::kFloat})); - - // Texture3D variants (fp16 output, half scales) - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_texture", - .dtype = vkapi::kHalf, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d_texture", - .dtype = vkapi::kHalf, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup_texture", - .dtype = vkapi::kHalf, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 50, - .embed_dim = 64, - .group_size = 64, - .indices_shape = {2, 4}, - .test_case_name = "single_group_texture", - .dtype = vkapi::kHalf, - .storage_type = utils::kTexture3D})); - - // Texture3D variants (fp32 output, half scales) - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_fp32_texture", - .dtype = vkapi::kFloat, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d_fp32_texture", - .dtype = vkapi::kFloat, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup_fp32_texture", - .dtype = vkapi::kFloat, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 50, - .embed_dim = 64, - .group_size = 64, - .indices_shape = {2, 4}, - .test_case_name = "single_group_fp32_texture", - .dtype = vkapi::kFloat, - .storage_type = utils::kTexture3D})); - - // --- Float scales --- - - // Buffer variants with float scales - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_float_scales", - .dtype = vkapi::kHalf, - .scales_dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d_float_scales", - .dtype = vkapi::kHalf, - .scales_dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup_float_scales", - .dtype = vkapi::kHalf, - .scales_dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_fp32_float_scales", - .dtype = vkapi::kFloat, - .scales_dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 32, - .embed_dim = 64, - .group_size = 32, - .indices_shape = {2, 3}, - .test_case_name = "small_2d_fp32_float_scales", - .dtype = vkapi::kFloat, - .scales_dtype = vkapi::kFloat})); - - test_cases.push_back(create_test_case( - {.vocab_size = 100, - .embed_dim = 128, - .group_size = 32, - .indices_shape = {4, 8}, - .test_case_name = "medium_multigroup_fp32_float_scales", - .dtype = vkapi::kFloat, - .scales_dtype = vkapi::kFloat})); - - // Texture3D with float scales - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_float_scales_texture", - .dtype = vkapi::kHalf, - .scales_dtype = vkapi::kFloat, - .storage_type = utils::kTexture3D})); - - test_cases.push_back(create_test_case( - {.vocab_size = 16, - .embed_dim = 32, - .group_size = 32, - .indices_shape = {4}, - .test_case_name = "small_1d_fp32_float_scales_texture", - .dtype = vkapi::kFloat, - .scales_dtype = vkapi::kFloat, - .storage_type = utils::kTexture3D})); - - // Llama 3.2 1B prefill configuration (fp32 output, half scales) - test_cases.push_back(create_test_case( - {.vocab_size = 128256, - .embed_dim = 2048, - .group_size = 32, - .indices_shape = {1, 2047}, - .test_case_name = "llama_3_2_1b_prefill_fp32", - .dtype = vkapi::kFloat, - .storage_type = utils::kBuffer})); - - // Llama 3.2 1B prefill configuration (fp32 output, float scales) - test_cases.push_back(create_test_case( - {.vocab_size = 128256, - .embed_dim = 2048, - .group_size = 32, - .indices_shape = {1, 2047}, - .test_case_name = "llama_3_2_1b_prefill_fp32_float_scales", - .dtype = vkapi::kFloat, - .scales_dtype = vkapi::kFloat, - .storage_type = utils::kBuffer})); - - // Llama 3.2 1B prefill configuration (fp16 output, half scales) - test_cases.push_back(create_test_case( - {.vocab_size = 128256, - .embed_dim = 2048, - .group_size = 32, - .indices_shape = {1, 2047}, - .test_case_name = "llama_3_2_1b_prefill_fp16", - .dtype = vkapi::kHalf, - .storage_type = utils::kBuffer})); - - // Llama 3.2 1B prefill configuration (fp16 output, float scales) - test_cases.push_back(create_test_case( - {.vocab_size = 128256, - .embed_dim = 2048, - .group_size = 32, - .indices_shape = {1, 2047}, - .test_case_name = "llama_3_2_1b_prefill_fp16_float_scales", - .dtype = vkapi::kHalf, - .scales_dtype = vkapi::kFloat, - .storage_type = utils::kBuffer})); - - return test_cases; -} - -int main(int argc, char** argv) { - auto results = execute_test_cases( - generate_test_cases, - "embedding_q4gsw", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - embedding_4bit_reference); - - results.print_summary(); - - if (results.get_failed_count() > 0) { - std::cerr << "FAILED: " << results.get_failed_count() << " test(s) failed." - << std::endl; - return 1; - } - - std::cout << "PASSED: All tests passed." << std::endl; - return 0; -} diff --git a/backends/vulkan/test/custom_ops/test_fpa_q4gsw_linear.cpp b/backends/vulkan/test/custom_ops/test_fpa_q4gsw_linear.cpp deleted file mode 100644 index 212ba03ddaa..00000000000 --- a/backends/vulkan/test/custom_ops/test_fpa_q4gsw_linear.cpp +++ /dev/null @@ -1,548 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -// FPA Q4GSW Linear A/B benchmark binary. -// -// Each generated test case has an `impl_selector` arg routed to the test -// op `test_etvk.test_fpa_q4gsw_linear.{gemm,gemv}` in TestFpaQ4gswLinear.cpp: -// -// GEMM (is_gemv=false): -// 0 -> PROD (et_vk.q4gsw_linear.default; dtype-based -// picker) 1 -> GEMM_W_4X8 (forced non-tin GEMM, nc buffer -// weight) 2 -> GEMM_TIN_W_4X8 (forced tin GEMM, nc buffer -// weight) 3 -> LEGACY (et_vk.linear_q4gsw.default -// legacy shaders) -// -// GEMV (is_gemv=true): -// 0 -> PROD (et_vk.q4gsw_linear.default; -// dtype-based picker) 1 -> GEMV_W_4X8 (forced gemv with -// subgroup broadcast) 2 -> GEMV_W_4X8_NOSG (forced gemv -// without subgroup broadcast) 3 -> LEGACY (et_vk.linear_q4gsw.default -// legacy shaders) 13 -> GEMV_COOP_W_4X8_NC_BUFFER (coop GEMV reusing the -// production -// nc-buffer prepack — same payload as -// W_4X8 GEMM/TIN GEMM/sg-GEMV; -// == g1w64 decomposition) -// 14 -> GEMV_COOP_..._G1W64 (force NUM_GROUPS=1, -// WORKERS_PER_GROUP=64) 15 -> GEMV_COOP_..._G4W16 (force -// NUM_GROUPS=4, WORKERS_PER_GROUP=16) 16 -> GEMV_COOP_..._G8W8 (force -// NUM_GROUPS=8, WORKERS_PER_GROUP=8) -// -// Selectors 14-16 pin the coop nc-buffer GEMV to an explicit reduction -// decomposition regardless of N. The production picker (pick_coop_variant_for_N -// in Q4gswLinear.cpp) only chooses g4w16 / g8w8 at PERF-sized N where the -// reference impl is skipped; these forced selectors give g4w16 / g8w8 numeric -// (ACCU) coverage at small N. Production picker behavior is unchanged. -// -// Selector 3 (LEGACY) is the in-prod q4gsw linear path. It uses a different -// prepack (pack_q4_linear_weight) and shader family -// (linear_q4gsw_tiled_* / linear_q4gsw_coop_*); the framework's per-shader -// timing breakdown will pick those up automatically. - -#include -#include -#include -#include -#include "utils.h" - -using namespace executorch::vulkan::prototyping; - -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 300; - -// Linear configuration struct. -struct LinearConfig { - int64_t M; - int64_t K; - int64_t N; - int64_t group_size; - bool has_bias = false; -}; - -// Convert a ValueSpec's input data (float or half) into a flat -// std::vector for use in the reference implementation. -static std::vector input_to_float_vec(const ValueSpec& spec) { - if (spec.dtype == vkapi::kFloat) { - return spec.get_float_data(); - } - if (spec.dtype == vkapi::kHalf) { - const auto& half_data = spec.get_half_data(); - std::vector out(half_data.size()); - for (size_t i = 0; i < half_data.size(); ++i) { - out[i] = half_to_float(half_data[i]); - } - return out; - } - throw std::invalid_argument( - "Reference implementation supports only float/half input dtypes."); -} - -// Create a single test case for the test_fpa_q4gsw_linear.{gemm,gemv} op. -TestCase create_test_case( - const LinearConfig& config, - vkapi::ScalarType dtype, - utils::StorageType storage, - int32_t impl_selector, - bool is_gemv) { - TestCase test_case; - - const int64_t M = config.M; - const int64_t K = config.K; - const int64_t N = config.N; - const int64_t group_size = config.group_size; - - const bool is_performance = - (M > kRefDimSizeLimit || K > kRefDimSizeLimit || N > kRefDimSizeLimit); - const std::string prefix = is_performance ? "PERF" : "ACCU"; - - const std::string dtype_str = dtype_short(dtype); - const std::string shape_str = shape_bracket({M, K}) + "x[" + - std::to_string(N) + "," + std::to_string(K) + "] g" + - std::to_string(group_size); - const std::string storage_str = repr_str(storage, utils::kWidthPacked); - std::string suffix = std::string("[") + (is_gemv ? "gemv" : "gemm") + " s" + - std::to_string(impl_selector) + "]"; - suffix += config.has_bias ? " bias" : " no_bias"; - const std::string test_name = make_test_label( - prefix, dtype_str, dtype_str, shape_str, storage_str, suffix); - test_case.set_name(test_name); - - const std::string op_name = is_gemv ? "test_etvk.test_fpa_q4gsw_linear.gemv" - : "test_etvk.test_fpa_q4gsw_linear.gemm"; - test_case.set_operator_name(op_name); - - // Input: [M, K] - ValueSpec input( - {M, K}, dtype, storage, utils::kWidthPacked, DataGenType::RANDINT); - - // Weight: [N, K/2] uint8 packed 4-bit - ValueSpec weight( - {N, K / 2}, - vkapi::kByte, - storage, - utils::kWidthPacked, - DataGenType::RANDINT4); - weight.set_constant(true); - weight.set_int4(true); - - // Scales: [K/gs, N] matching input dtype (the custom op prepacks scales - // using the input tensor's dtype). - ValueSpec scales( - {K / group_size, N}, - dtype, - storage, - utils::kWidthPacked, - DataGenType::RANDOM_SCALES); - scales.set_constant(true); - - // Group size - ValueSpec gs_spec(static_cast(group_size)); - - // Bias - ValueSpec bias( - {N}, - dtype, - storage, - utils::kWidthPacked, - config.has_bias ? DataGenType::RANDOM : DataGenType::ZEROS); - bias.set_constant(true); - if (!config.has_bias) { - bias.set_none(true); - } - - // impl_selector as int32 - ValueSpec impl_selector_spec(static_cast(impl_selector)); - - // Output: [M, N] - ValueSpec output( - {M, N}, dtype, storage, utils::kWidthPacked, DataGenType::ZEROS); - - // Tolerance: fp16 outputs use relaxed tolerance to account for f16 - // accumulation / rounding. - float base_tol = 0.05f * (static_cast(K) / 64.0f); - float tol = (dtype == vkapi::kHalf) ? (4.0f * base_tol) : base_tol; - test_case.set_abs_tolerance(tol); - - test_case.add_input_spec(input); - test_case.add_input_spec(weight); - test_case.add_input_spec(scales); - test_case.add_input_spec(gs_spec); - test_case.add_input_spec(bias); - test_case.add_input_spec(impl_selector_spec); - test_case.add_output_spec(output); - - return test_case; -} - -// Reference implementation: simple dequant + fp32 GEMM. Only runs for -// small shapes (gate on kRefDimSizeLimit). -void linear_q4gsw_reference_impl(TestCase& test_case) { - int32_t idx = 0; - const ValueSpec& input_spec = test_case.inputs()[idx++]; - const ValueSpec& weight_spec = test_case.inputs()[idx++]; - const ValueSpec& scales_spec = test_case.inputs()[idx++]; - const ValueSpec& gs_spec = test_case.inputs()[idx++]; - const ValueSpec& bias_spec = test_case.inputs()[idx++]; - // impl_selector is not used in the reference impl - ++idx; - - ValueSpec& output_spec = test_case.outputs()[0]; - - auto input_sizes = input_spec.get_tensor_sizes(); - auto output_sizes = output_spec.get_tensor_sizes(); - - int64_t M = input_sizes[0]; - int64_t K = input_sizes[1]; - int64_t N = output_sizes[1]; - int64_t group_size = gs_spec.get_int_value(); - - if (M > kRefDimSizeLimit || K > kRefDimSizeLimit || N > kRefDimSizeLimit) { - throw std::invalid_argument( - "Dimensions exceed limit for reference implementation."); - } - - std::vector input_data = input_to_float_vec(input_spec); - auto& weight_data = weight_spec.get_uint8_data(); - std::vector scales_data = input_to_float_vec(scales_spec); - std::vector bias_data; - if (!bias_spec.is_none()) { - bias_data = input_to_float_vec(bias_spec); - } - - int64_t num_output_elements = M * N; - auto& ref_data = output_spec.get_ref_float_data(); - ref_data.resize(num_output_elements); - - for (int64_t m = 0; m < M; ++m) { - for (int64_t n = 0; n < N; ++n) { - float sum = 0.0f; - for (int64_t k = 0; k < K; ++k) { - float input_val = input_data[m * K + k]; - - int64_t weight_idx = n * (K / 2) + (k / 2); - uint8_t packed = weight_data[weight_idx]; - int8_t nibble = (k % 2 == 0) - ? static_cast(packed & 0x0F) - 8 - : static_cast((packed >> 4) & 0x0F) - 8; - - int64_t group_idx = k / group_size; - float scale = scales_data[group_idx * N + n]; - - sum += input_val * static_cast(nibble) * scale; - } - if (!bias_spec.is_none()) { - sum += bias_data[n]; - } - ref_data[m * N + n] = sum; - } - } -} - -void reference_impl(TestCase& test_case) { - linear_q4gsw_reference_impl(test_case); -} - -// Custom FLOP calculator: 2 * M * K * N for the linear op itself. -int64_t linear_flop_calculator(const TestCase& test_case) { - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - - int64_t M = input_sizes[0]; - int64_t K = input_sizes[1]; - int64_t N = output_sizes[1]; - return 2 * M * K * N; -} - -// Canonical N/K shapes for LLM hidden-size sweeps. -static const std::vector>& get_nk_shapes() { - static const std::vector> kShapes = { - // (K, N) - {1024, 2048}, - {4096, 4096}, - // {4096, 14336}, // Large-N case can make the full benchmark binary - // unstable. - }; - return kShapes; -} - -// GEMM sweep test cases: M in {32, 128, 256} x N/K shapes x dtype x storage -// x impl_selector in kGemmSelectors. -// -// Selector 3 is the legacy in-prod q4gsw linear path -// (et_vk.linear_q4gsw.default) registered in QuantizedLinear.cpp. -std::vector generate_gemm_test_cases() { - std::vector test_cases; - - const std::vector gemm_Ms = {32, 128, 256}; - const int64_t group_size = 32; - - const std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; - const std::vector storages = { - utils::kBuffer, utils::kTexture3D}; - - // Selectors exercised in the GEMM PERF/ACCU sweep: PROD (0), forced non-tin - // / tin GEMM (1, 2), legacy (3). - const std::vector kGemmSelectors = {0, 1, 2, 3}; - - for (int64_t M : gemm_Ms) { - for (const auto& shape : get_nk_shapes()) { - const int64_t K = shape.first; - const int64_t N = shape.second; - LinearConfig cfg{M, K, N, group_size}; - for (auto dtype : dtypes) { - for (auto storage : storages) { - for (int32_t selector : kGemmSelectors) { - test_cases.push_back(create_test_case( - cfg, dtype, storage, selector, /*is_gemv=*/false)); - } - } - } - } - } - - // Non-aligned-N coverage for the W_4X8 GEMM path. The fp32 GEMM issues a - // 16B ivec4 weight load that spans two consecutive (k4, n4) ivec2 tiles - // along N, so N4 must be even (== N a multiple of 8) at the buffer-stride - // level. The prepack pads the weight buffer's row stride to next-even N4 - // and fills the OOB tiles with bias-zero nibbles; these accuracy cases - // exercise that padding path on shapes with N % 8 != 0. Only the new W_4X8 - // family is tested (selectors 0, 1, 2, 5, 6) — selector 3 (LEGACY) uses a - // different prepack and supports arbitrary N. - const std::vector> kNonAlignedNkShapes = { - // (K, N) — K kept under kRefDimSizeLimit so reference impl runs. - {128, 12}, - {128, 20}, - }; - const std::vector kNonAlignedSelectors = {0, 1, 2}; - for (const auto& shape : kNonAlignedNkShapes) { - const int64_t K = shape.first; - const int64_t N = shape.second; - LinearConfig cfg{32, K, N, group_size}; - for (auto dtype : dtypes) { - for (auto storage : storages) { - for (int32_t selector : kNonAlignedSelectors) { - test_cases.push_back(create_test_case( - cfg, dtype, storage, selector, /*is_gemv=*/false)); - } - } - } - } - - // Small ACCU shape (M=32, K=128, N=128) under kRefDimSizeLimit so the - // reference impl runs. Sanity-checks GEMM correctness during iteration. - { - LinearConfig cfg{32, 128, 128, group_size}; - for (auto dtype : {vkapi::kFloat, vkapi::kHalf}) { - test_cases.push_back(create_test_case( - cfg, - dtype, - utils::kTexture3D, - /*impl_selector=*/0, - /*is_gemv=*/false)); - } - } - - // M-tail ACCU shapes. These exercise final partial GEMM tiles for both the - // fp32 direct-input path (tile height 4) and the fp16 TIN path (tile height - // 8). - for (int64_t M : {31, 33}) { - LinearConfig cfg{M, 128, 128, group_size}; - for (auto dtype : {vkapi::kFloat, vkapi::kHalf}) { - test_cases.push_back(create_test_case( - cfg, - dtype, - utils::kTexture3D, - /*impl_selector=*/0, - /*is_gemv=*/false)); - } - } - - return test_cases; -} - -// GEMV sweep test cases: M = 1 x N/K shapes x dtype x storage x -// impl_selector in {0, 1, 2, 3}. -// -// Selector 3 is the legacy in-prod q4gsw linear path -// (et_vk.linear_q4gsw.default) registered in QuantizedLinear.cpp. -std::vector generate_gemv_test_cases() { - std::vector test_cases; - - const int64_t group_size = 32; - - const std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; - const std::vector storages = { - utils::kBuffer, utils::kTexture3D}; - - // ACCU correctness shapes (under kRefDimSizeLimit=300). Exercise selectors - // PROD (0) and forced nosg (2). Forced sg (1) is intentionally skipped: - // sg requires subgroupSize==64 and produces incorrect results on Mali - // (subgroupSize==16); on those devices the PROD picker correctly routes - // to the nosg variant. N must be a multiple of 128 (= 2 * LWG.x) so the - // GEMV shader has no early-exit threads in any workgroup. - const std::vector> kAccuShapes = { - // (K, N) - {128, 128}, - {256, 256}, - }; - // Selector 13 (nc Buffer, reuses production prepack) included for ACCU - // coverage of the coop nc weight-binding variant. - const std::vector kAccuSelectors = {0, 2, 13}; - for (const auto& shape : kAccuShapes) { - const int64_t K = shape.first; - const int64_t N = shape.second; - LinearConfig cfg{1, K, N, group_size}; - for (auto dtype : dtypes) { - for (auto storage : storages) { - for (int32_t selector : kAccuSelectors) { - test_cases.push_back(create_test_case( - cfg, dtype, storage, selector, /*is_gemv=*/true)); - } - } - } - } - - // Forced coop-reduction-decomposition ACCU coverage (selectors 14/15/16 = - // g1w64 / g4w16 / g8w8). The production picker (pick_coop_variant_for_N) - // only selects g4w16 (10244096) at PERF-sized N, where - // every dim exceeds kRefDimSizeLimit=300 so the reference impl is skipped and - // those reduction decompositions get zero numeric validation. These cases - // pin each decomposition regardless of N at small shapes (all dims <= 300) - // so the reference runs and proves g4w16 / g8w8 compute the SAME result as - // the reference — the M=1 decode path actually shipped for Qwen3 / Llama. - // - // Each WG of variant gN produces N*8 outputs (g1w64 -> 8, g4w16 -> 32, - // g8w8 -> 64), so the N values below tile cleanly into all three: N=128 - // (16 / 4 / 2 WGs) and N=256 (32 / 8 / 4 WGs). The shader also handles a - // ragged final WG, but clean tiles keep the test intent unambiguous. K is - // swept over {64, 128, 256} (all multiples of group_size=32 and <= 300) so - // the K-loop reduction is exercised across short and longer accumulations. - const std::vector> kCoopForcedAccuShapes = { - // (K, N) — all dims <= 300 so linear_q4gsw_reference_impl runs. - {64, 128}, - {128, 256}, - {256, 128}, - }; - // 14 -> g1w64, 15 -> g4w16, 16 -> g8w8. g1w64 included for symmetry. - const std::vector kCoopForcedSelectors = {14, 15, 16}; - for (const auto& shape : kCoopForcedAccuShapes) { - const int64_t K = shape.first; - const int64_t N = shape.second; - LinearConfig cfg{1, K, N, group_size}; - for (auto dtype : dtypes) { - for (auto storage : storages) { - for (int32_t selector : kCoopForcedSelectors) { - test_cases.push_back(create_test_case( - cfg, dtype, storage, selector, /*is_gemv=*/true)); - } - } - } - } - - // GEMV PERF selectors: PROD (0), forced sg (1), forced nosg (2), LEGACY (3), - // nc-Buffer coop (13, reuses production prepack — single-format - // prefill+decode). - const std::vector kGemvPerfSelectors = {0, 1, 2, 3, 13}; - for (const auto& shape : get_nk_shapes()) { - const int64_t K = shape.first; - const int64_t N = shape.second; - LinearConfig cfg{1, K, N, group_size}; - for (auto dtype : dtypes) { - for (auto storage : storages) { - for (int32_t selector : kGemvPerfSelectors) { - test_cases.push_back(create_test_case( - cfg, dtype, storage, selector, /*is_gemv=*/true)); - } - } - } - } - - // LLM-decode-shape PERF cells (M=1 GEMV, group_size=32). Mirrors the actual - // per-layer linear shapes seen during decode profiling on Llama 3.2 1B and - // Qwen3 0.6B; the original Phase 2 corpus (1024/2048/4096 x 2048/4096/11008) - // under-samples these and missed the regression where sg-GEMV (selector 1) - // is 15-22% slower per dispatch than LEGACY coop (selector 3) on Adreno 750. - // - // All N values here are multiples of 128 (= 2 * LWG.x for the GEMV shader), - // so the GEMV shader has no early-exit threads. N=512 is the Llama 3.2 1B - // k_proj/v_proj projection (GQA) and is a multiple of 4 (the prepack - // requirement: prepack_q4_w_4x8_nc_buffer enforces N % 4 == 0). - // - // Default storage is fp16 + Tex3D - that's the actual decode config and the - // shape combo where the regression was observed. We additionally exercise - // K=2048,N=2048 under fp32 + Tex3D and fp32 + Buffer to confirm the - // regression isn't fp16-Tex3D-specific. All four selectors (PROD, sg, nosg, - // LEGACY) are exercised. - const std::vector> kLlmGemvShapes = { - // (K, N) - Llama 3.2 1B - {2048, 512}, // k_proj / v_proj (GQA) - {2048, 2048}, // q_proj - {2048, 8192}, // gate_proj / up_proj - {8192, 2048}, // down_proj - // (K, N) - Qwen3 0.6B - {1024, 1024}, // k_proj / v_proj - {1024, 2048}, // q_proj (also overlaps with original corpus) - {1024, 3072}, // gate_proj / up_proj - {3072, 1024}, // down_proj - }; - for (const auto& shape : kLlmGemvShapes) { - const int64_t K = shape.first; - const int64_t N = shape.second; - LinearConfig cfg{1, K, N, group_size}; - for (int32_t selector : kGemvPerfSelectors) { - test_cases.push_back(create_test_case( - cfg, vkapi::kHalf, utils::kTexture3D, selector, /*is_gemv=*/true)); - } - } - // Diversity sanity check: K=2048,N=2048 under fp32 + {Tex3D, Buffer} to - // confirm the regression isn't fp16-Tex3D-specific. - { - LinearConfig cfg{1, 2048, 2048, group_size}; - for (auto storage : {utils::kTexture3D, utils::kBuffer}) { - for (int32_t selector : kGemvPerfSelectors) { - test_cases.push_back(create_test_case( - cfg, vkapi::kFloat, storage, selector, /*is_gemv=*/true)); - } - } - } - - return test_cases; -} - -std::vector generate_all_test_cases() { - auto gemv = generate_gemv_test_cases(); - auto gemm = generate_gemm_test_cases(); - gemv.insert(gemv.end(), gemm.begin(), gemm.end()); - return gemv; -} - -int main(int argc, char* argv[]) { - (void)argc; - (void)argv; - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "FPA Q4GSW Linear A/B Variant Prototyping Framework (gemm + gemv)" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_all_test_cases, - linear_flop_calculator, - "FpaQ4gswLinear", - 3, - 10, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/custom_ops/test_q8ta_conv2d.cpp b/backends/vulkan/test/custom_ops/test_q8ta_conv2d.cpp deleted file mode 100644 index 90fd4e1616b..00000000000 --- a/backends/vulkan/test/custom_ops/test_q8ta_conv2d.cpp +++ /dev/null @@ -1,1593 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include "conv2d_utils.h" -#include "utils.h" - -// #define DEBUG_MODE - -using namespace executorch::vulkan::prototyping; - -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 100; -static constexpr int64_t kRefOperationLimit = 2 * 1024 * 1024; - -struct Im2colUnsignedTestOptions { - int32_t input_zero_point = 2; - int32_t output_zero_point = -1; - bool use_extreme_values = false; - bool use_accumulator_limit_values = false; - bool has_bias = true; - const char* activation = "none"; - float weight_scale = 1.0f / 256.0f; -}; - -// Utility function to create a test case from a Conv2dConfig -static TestCase create_test_case_from_config_with_layouts( - const Conv2dConfig& config, - vkapi::ScalarType input_dtype, - utils::StorageType fp_storage_type, - utils::GPUMemoryLayout input_int8_memory_layout, - utils::GPUMemoryLayout output_int8_memory_layout, - const std::string& impl_selector = "", - const Im2colUnsignedTestOptions* im2col_options = nullptr, - const float input_scale_val = 0.008123f, - const DataGenType input_data_gen = DataGenType::RANDOM) { - TestCase test_case; - - // Calculate output dimensions - int64_t H_out = config.get_output_height(); - int64_t W_out = config.get_output_width(); - - // Input tensor (float/half) - [N, C_in, H_in, W_in] - std::vector input_size = { - config.batch, - config.channels.in, - config.input_size.h, - config.input_size.w}; - - utils::GPUMemoryLayout fp_memory_layout = fp_storage_type == utils::kBuffer - ? utils::kWidthPacked - : utils::kChannelsPacked; - - // Create test case name - std::string prefix = config.test_case_name.substr(0, 4); // "ACCU" or "PERF" - std::string dtype_str = dtype_short(input_dtype); - std::string in_shape = "[" + std::to_string(config.batch) + "," + - std::to_string(config.channels.in) + "," + - std::to_string(config.input_size.h) + "," + - std::to_string(config.input_size.w) + "]"; - std::string weight_shape = "[" + std::to_string(config.channels.out) + "," + - std::to_string(config.channels.in / config.groups) + "," + - std::to_string(config.kernel.h) + "," + std::to_string(config.kernel.w) + - "]"; - std::string shape_str = in_shape + "x" + weight_shape + " s" + - std::to_string(config.stride.h) + " p" + - std::to_string(config.padding.h) + " d" + - std::to_string(config.dilation.h) + " g" + std::to_string(config.groups); - std::string storage_str = repr_str(utils::kBuffer, input_int8_memory_layout); - if (input_int8_memory_layout != output_int8_memory_layout) { - storage_str += "->" + repr_str(utils::kBuffer, output_int8_memory_layout); - } - std::string suffix = impl_selector.empty() ? "" : "[" + impl_selector + "]"; - std::string test_name = make_test_label( - prefix, dtype_str, dtype_str, shape_str, storage_str, suffix); - test_case.set_name(test_name); - - // Set the operator name for the test case - use the unified test operator - std::string operator_name = "test_etvk.test_q8ta_conv2d.default"; - test_case.set_operator_name(operator_name); - - ValueSpec input_tensor( - input_size, - input_dtype, - fp_storage_type, - fp_memory_layout, - input_data_gen); - - if (debugging()) { - print_valuespec_data(input_tensor, "input_tensor"); - } - - ValueSpec input_scale(input_scale_val); - - const int32_t input_zero_point_val = - im2col_options == nullptr ? 2 : im2col_options->input_zero_point; - ValueSpec input_zero_point(input_zero_point_val); - - if (im2col_options != nullptr && - im2col_options->use_accumulator_limit_values) { - input_tensor.ensure_data_generated(2401); - std::fill( - input_tensor.get_float_data().begin(), - input_tensor.get_float_data().end(), - (127.0f - input_zero_point_val) * input_scale_val); - } else if (im2col_options != nullptr && im2col_options->use_extreme_values) { - input_tensor.ensure_data_generated(2401); - constexpr std::array values = {-128, -1, 0, 1, 127}; - std::vector& input_data = input_tensor.get_float_data(); - for (size_t i = 0; i < input_data.size(); ++i) { - input_data.at(i) = (static_cast(values.at(i % values.size())) - - input_zero_point_val) * - input_scale_val; - } - } - - // Quantized weight tensor (int8) - [C_out, C_in_per_group * K_h * K_w] - // Memory layout: height, width, then channels - in_c is innermost (stride 1) - // in the second dimension - const int64_t in_channels_per_group = config.channels.in / config.groups; - const int64_t in_features = utils::align_up_4( - in_channels_per_group * config.kernel.h * config.kernel.w); - std::vector weight_size = {config.channels.out, in_features}; - ValueSpec quantized_weight( - weight_size, - vkapi::kChar, // int8 for quantized weights - fp_storage_type, - utils::kWidthPacked, - DataGenType::RANDINT8); - quantized_weight.set_constant(true); - - if (im2col_options != nullptr && - im2col_options->use_accumulator_limit_values) { - quantized_weight.ensure_data_generated(2402); - std::fill( - quantized_weight.get_int8_data().begin(), - quantized_weight.get_int8_data().end(), - 127); - } else if (im2col_options != nullptr && im2col_options->use_extreme_values) { - quantized_weight.ensure_data_generated(2402); - constexpr std::array values = {-128, -1, 0, 1, 127}; - std::vector& weight_data = quantized_weight.get_int8_data(); - for (size_t i = 0; i < weight_data.size(); ++i) { - weight_data.at(i) = values.at((i * 3 + 1) % values.size()); - } - } - - if (debugging()) { - print_valuespec_data(quantized_weight, "weight_tensor"); - } - - const int64_t aligned_out_channels = utils::align_up_4(config.channels.out); - - // Weight quantization scales (float/half, per-channel) - ValueSpec weight_scales( - {aligned_out_channels}, // Per output channel - input_dtype, - fp_storage_type, - utils::kWidthPacked, - DataGenType::RANDOM_SCALES); - weight_scales.set_constant(true); - if (im2col_options != nullptr) { - weight_scales.ensure_data_generated(2403); - std::fill( - weight_scales.get_float_data().begin(), - weight_scales.get_float_data().end(), - im2col_options->weight_scale); - } - - ValueSpec weight_sums( - {aligned_out_channels}, // Per output channel - vkapi::kInt, - fp_storage_type, - utils::kWidthPacked, - DataGenType::ZEROS); - weight_sums.set_constant(true); - - // Compute weight_sums data based on quantized weights - compute_weight_sums( - weight_sums, quantized_weight, config.channels.out, in_features); - - // Bias (optional, float/half) - [C_out] - ValueSpec bias( - {aligned_out_channels}, // Per output channel - input_dtype, - fp_storage_type, - utils::kWidthPacked, - DataGenType::ZEROS); - bias.set_constant(true); - if (im2col_options != nullptr && !im2col_options->has_bias) { - bias.set_none(true); - } - - // Output quantization parameters - float output_scale_val = 0.05314; - ValueSpec output_scale(output_scale_val); - - const int32_t output_zero_point_val = - im2col_options == nullptr ? -1 : im2col_options->output_zero_point; - ValueSpec output_zero_point(output_zero_point_val); - - // Stride and padding parameters - ValueSpec stride({config.stride.h, config.stride.w}); - ValueSpec padding({config.padding.h, config.padding.w}); - - // Dilation and groups parameters - ValueSpec dilation({config.dilation.h, config.dilation.w}); - ValueSpec groups(config.groups); - - // Kernel size parameters - ValueSpec kernel_size({config.kernel.h, config.kernel.w}); - - // Output tensor (float/half) - [N, C_out, H_out, W_out] - ValueSpec output( - {config.batch, config.channels.out, H_out, W_out}, - input_dtype, - fp_storage_type, - fp_memory_layout, - DataGenType::ZEROS); - - // Add all specs to test case for q8ta_q8csw_q8to operation - test_case.add_input_spec(input_tensor); - test_case.add_input_spec(input_scale); - test_case.add_input_spec(input_zero_point); - test_case.add_input_spec(quantized_weight); - test_case.add_input_spec(weight_sums); - test_case.add_input_spec(weight_scales); - test_case.add_input_spec(output_scale); - test_case.add_input_spec(output_zero_point); - test_case.add_input_spec(bias); - test_case.add_input_spec(kernel_size); - test_case.add_input_spec(stride); - test_case.add_input_spec(padding); - test_case.add_input_spec(dilation); - test_case.add_input_spec(groups); - - // Activation (none = no activation) - ValueSpec activation = ValueSpec::make_string( - im2col_options == nullptr ? "none" : im2col_options->activation); - test_case.add_input_spec(activation); - - // Add memory layout parameter for the quantized tensors - ValueSpec input_layout_int(static_cast(input_int8_memory_layout)); - test_case.add_input_spec(input_layout_int); - - ValueSpec output_layout_int(static_cast(output_int8_memory_layout)); - test_case.add_input_spec(output_layout_int); - - // Add impl_selector string - ValueSpec impl_selector_spec = ValueSpec::make_string(impl_selector); - test_case.add_input_spec(impl_selector_spec); - - test_case.add_output_spec(output); - - test_case.set_abs_tolerance( - im2col_options == nullptr ? output_scale_val + 1e-4f - : output_scale_val * 0.25f); - - // Filter out quantize/dequantize shaders from timing measurements - test_case.set_shader_filter({ - "nchw_to", - "to_nchw", - "q8ta_quantize", - "q8ta_dequantize", - }); - - return test_case; -} - -static TestCase create_test_case_from_config( - const Conv2dConfig& config, - vkapi::ScalarType input_dtype, - utils::StorageType fp_storage_type, - utils::GPUMemoryLayout int8_memory_layout, - const std::string& impl_selector = "", - const Im2colUnsignedTestOptions* im2col_options = nullptr, - const float input_scale_val = 0.008123f, - const DataGenType input_data_gen = DataGenType::RANDOM) { - return create_test_case_from_config_with_layouts( - config, - input_dtype, - fp_storage_type, - int8_memory_layout, - int8_memory_layout, - impl_selector, - im2col_options, - input_scale_val, - input_data_gen); -} - -static std::vector generate_narrow_workgroup_test_cases() { - std::vector test_cases; - std::vector configs = { - {OutInChannels(64, 32), - InputSize2D(9, 9), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(128, 32), - InputSize2D(7, 7), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - }; - - for (auto& config : configs) { - const bool is_performance = config.channels.out > kRefDimSizeLimit; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = make_test_case_name( - config, is_performance, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4C, - /*impl_selector=*/"general")); - } - return test_cases; -} - -// Generate easy test cases for quantized conv2d operation (for debugging) -std::vector generate_quantized_conv2d_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - Conv2dConfig config = { - OutInChannels(16, 8), // channels (out, in) - InputSize2D(5, 5), // input_size (h, w) - KernelSize(3, 3), // kernel - Stride(1, 1), // stride - Padding(1, 1), // padding - Dilation(1, 1), // dilation - 1, // groups - }; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - - std::vector fp_storage_types = {utils::kTexture3D}; - - // Memory layouts for int8 tensors - test both optimized (4W4C) and general - // paths - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - // Generate test cases for each combination - for (const utils::StorageType fp_storage_type : fp_storage_types) { - for (const utils::GPUMemoryLayout int8_memory_layout : - int8_memory_layouts) { - config.test_case_name = - make_test_case_name(config, false, fp_storage_type, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); - - // Test im2col implementation when input channels per group is a - // multiple of 4 - if ((config.channels.in / config.groups) % 4 == 0) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"im2col")); - } - // For 4W4C layout, also test the legacy implementation - if (int8_memory_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"legacy_4w4c")); - } - } - } - - return test_cases; -} - -static std::vector generate_im2col_unsigned_test_cases( - const std::string& impl_selector); - -static std::vector generate_streaming_im2col_test_cases() { - std::vector test_cases; - - Conv2dConfig full_fit_config = { - OutInChannels(4, 32), - InputSize2D(30, 99), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1, - 10}; - full_fit_config.op_name = "conv2d_q8ta_q8csw_q8to"; - full_fit_config.test_case_name = make_test_case_name( - full_fit_config, false, utils::kTexture3D, utils::kBuffer); - - Conv2dConfig streaming_fallback_config = full_fit_config; - streaming_fallback_config.channels.in = 64; - streaming_fallback_config.test_case_name = make_test_case_name( - streaming_fallback_config, false, utils::kTexture3D, utils::kBuffer); - - // Forced-fallback cases need no int8 dot-product support, so they run on - // all devices; the kAuto cases below stay gated. - test_cases.push_back(create_test_case_from_config( - full_fit_config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - /*impl_selector=*/"im2col_fallback", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT)); - test_cases.push_back(create_test_case_from_config( - streaming_fallback_config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - /*impl_selector=*/"im2col_fallback", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT)); - - if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { - return test_cases; - } - - for (const utils::GPUMemoryLayout layout : - {utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C}) { - test_cases.push_back(create_test_case_from_config( - full_fit_config, - vkapi::kFloat, - utils::kTexture3D, - layout, - /*impl_selector=*/"im2col_auto", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT)); - } - Conv2dConfig streaming_config = full_fit_config; - streaming_config.channels.in = 64; - streaming_config.test_case_name = make_test_case_name( - streaming_config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - streaming_config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4C1W, - /*impl_selector=*/"im2col_auto", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT)); - Conv2dConfig grouped_config = { - OutInChannels(16, 32), - InputSize2D(30, 99), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2, - 20}; - grouped_config.op_name = "conv2d_q8ta_q8csw_q8to"; - grouped_config.test_case_name = make_test_case_name( - grouped_config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - grouped_config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - /*impl_selector=*/"im2col_auto", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT)); - - Conv2dConfig output_channel_tail_config = { - OutInChannels(10, 32), - InputSize2D(30, 99), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1, - 20}; - output_channel_tail_config.op_name = "conv2d_q8ta_q8csw_q8to"; - output_channel_tail_config.test_case_name = make_test_case_name( - output_channel_tail_config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - output_channel_tail_config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - /*impl_selector=*/"im2col_auto", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT)); - return test_cases; -} - -// SceneX route tests. The kPackedInt8_4C input + kPackedInt8_4W4C output -// layout combination is only exercised here (default generators never pair -// them); run via --scenex-regular . Zero -// tolerances are exact by construction: both pipelines accumulate in int32 -// with identical requantize, so any mismatch is a real regression, not noise. -static TestCase create_scenex_test_case( - const Conv2dConfig& source_config, - const std::string& route) { - Conv2dConfig config = source_config; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = - make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); - - const std::string impl_selector = route == "auto" ? "" - : route == "direct" ? "general" - : "im2col"; - TestCase test_case = create_test_case_from_config_with_layouts( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - impl_selector, - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT); - test_case.set_abs_tolerance(0.0f); - test_case.set_rel_tolerance(0.0f); - return test_case; -} - -static TestCase generate_scenex_regular_test_case( - const std::string& route, - const int case_index) { - const std::vector configs = { - {OutInChannels(128, 64), - InputSize2D(40, 51), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1, - 60}, - {OutInChannels(256, 128), - InputSize2D(20, 26), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1, - 60}, - }; - return create_scenex_test_case(configs.at(case_index), route); -} - -static TestCase generate_scenex_grouped_test_case( - const std::string& route, - const int case_index) { - const std::vector configs = { - {OutInChannels(64, 64), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2, - 60}, - {OutInChannels(128, 128), - InputSize2D(128, 128), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 4, - 60}, - {OutInChannels(64, 64), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2, - 60}, - }; - - return create_scenex_test_case(configs.at(case_index), route); -} - -// Generate test cases for quantized conv2d operation -static std::vector generate_quantized_conv2d_test_cases() { - std::vector test_cases; - api::Context* const context = vkcompute::api::context(); - if (!context->adapter_ptr()->supports_int8_dot_product()) { - for (const std::string& impl_selector : {"im2col", "im2col_auto"}) { - std::vector im2col_cases = - generate_im2col_unsigned_test_cases(impl_selector); - for (TestCase& test_case : im2col_cases) { - test_cases.push_back(std::move(test_case)); - } - } - return test_cases; - } - - std::vector configs = { - // General 2D convolutions - {OutInChannels(32, 3), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(32, 3), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(8, 8), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(16, 32), - InputSize2D(77, 77), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - // Grouped convolutions - {OutInChannels(64, 32), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2}, - {OutInChannels(96, 96), - InputSize2D(81, 81), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 3}, - {OutInChannels(96, 96), - InputSize2D(64, 64), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 4}, - // Performance cases (3x3 convs - will use im2col) - {OutInChannels(32, 3), - InputSize2D(256, 256), - KernelSize(3, 3), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(64, 64), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - // Performance cases (grouped convs) - {OutInChannels(64, 64), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2}, - {OutInChannels(96, 96), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 3}, - {OutInChannels(128, 128), - InputSize2D(128, 128), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 4}, - // SceneX v9 grouped convolutions (large spatial) - {OutInChannels(128, 128), - InputSize2D(256, 256), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 4}, - {OutInChannels(64, 64), - InputSize2D(256, 256), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2}, - // Deep channels + small spatial (ResNet50 stage 5 bottleneck) - {OutInChannels(512, 512), - InputSize2D(7, 7), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - // Strided 1x1 shortcut (worst-case strided downsample) - {OutInChannels(2048, 1024), - InputSize2D(14, 14), - KernelSize(1, 1), - Stride(2, 2), - Padding(0, 0), - Dilation(1, 1), - 1}}; - - // Test with different storage types and memory layouts - std::vector fp_storage_types = {utils::kTexture3D}; - - // Memory layouts for int8 tensors - test both optimized (4W4C) and general - // paths - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - // Generate test cases for each combination - for (auto& config : configs) { - bool is_performance = config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit || - config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - - config.op_name = "conv2d_q8ta_q8csw_q8to"; - - for (const utils::StorageType fp_storage_type : fp_storage_types) { - for (const utils::GPUMemoryLayout int8_memory_layout : - int8_memory_layouts) { - config.test_case_name = make_test_case_name( - config, is_performance, fp_storage_type, utils::kBuffer); - - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"general")); - - // Test im2col implementation when input channels per group is a - // multiple of 4 - const int64_t in_channels_per_group = - config.channels.in / config.groups; - if (in_channels_per_group % 4 == 0) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"im2col")); - } - - // For 4W4C layout, also test the legacy implementation - if (int8_memory_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"legacy_4w4c")); - } - - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); - } - } - } - - std::vector batch_configs = { - {OutInChannels(16, 32), - InputSize2D(7, 7), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1, - 2}, - {OutInChannels(32, 3), - InputSize2D(256, 256), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1, - 1}, - {OutInChannels(32, 3), - InputSize2D(256, 256), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1, - 60}, - {OutInChannels(512, 256), - InputSize2D(10, 13), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1, - 60}}; - - for (auto& config : batch_configs) { - const bool is_performance = config.batch > kRefDimSizeLimit || - config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit || - config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = make_test_case_name( - config, is_performance, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4C1W)); - if (config.batch == 2) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4C1W, - /*impl_selector=*/"im2col")); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4W4C)); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - /*impl_selector=*/"im2col")); - } - } - - for (const std::string& impl_selector : - {"im2col", "im2col_unsigned", "im2col_auto"}) { - std::vector im2col_cases = - generate_im2col_unsigned_test_cases(impl_selector); - for (TestCase& test_case : im2col_cases) { - test_cases.push_back(std::move(test_case)); - } - } - - auto narrow_workgroup_cases = generate_narrow_workgroup_test_cases(); - test_cases.insert( - test_cases.end(), - narrow_workgroup_cases.begin(), - narrow_workgroup_cases.end()); - - auto streaming_cases = generate_streaming_im2col_test_cases(); - test_cases.insert( - test_cases.end(), streaming_cases.begin(), streaming_cases.end()); - - return test_cases; -} - -static std::vector generate_im2col_unsigned_test_cases( - const std::string& impl_selector) { - api::Context* const context = vkcompute::api::context(); - if (impl_selector == "im2col_unsigned" && - !context->adapter_ptr()->supports_int8_dot_product()) { - return {}; - } - - std::vector> configs = { - {{OutInChannels(5, 4), - InputSize2D(5, 5), - KernelSize(3, 3), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {.input_zero_point = 2, - .output_zero_point = -1, - .use_extreme_values = true, - .use_accumulator_limit_values = false, - .has_bias = true, - .activation = "none"}}, - {{OutInChannels(8, 8), - InputSize2D(5, 5), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {.input_zero_point = -7, - .output_zero_point = 3, - .use_extreme_values = true, - .use_accumulator_limit_values = false, - .has_bias = false, - .activation = "none"}}, - {{OutInChannels(12, 8), - InputSize2D(7, 7), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - {.input_zero_point = 127, - .output_zero_point = -5, - .use_extreme_values = true, - .use_accumulator_limit_values = false, - .has_bias = true, - .activation = "relu"}}, - {{OutInChannels(12, 8), - InputSize2D(9, 9), - KernelSize(3, 3), - Stride(1, 1), - Padding(2, 2), - Dilation(2, 2), - 1}, - {.input_zero_point = -128, - .output_zero_point = 5, - .use_extreme_values = true, - .use_accumulator_limit_values = false, - .has_bias = false, - .activation = "none"}}, - {{OutInChannels(8, 8), - InputSize2D(6, 7), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 2}, - {.input_zero_point = 11, - .output_zero_point = -3, - .use_extreme_values = true, - .use_accumulator_limit_values = false, - .has_bias = true, - .activation = "relu"}}, - {{OutInChannels(1, 4), - InputSize2D(90, 91), - KernelSize(90, 91), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {.input_zero_point = 0, - .output_zero_point = -1, - .use_extreme_values = false, - .use_accumulator_limit_values = true, - .has_bias = true, - .activation = "none", - .weight_scale = 1.0f / 1000000.0f}}, - }; - - std::vector test_cases; - test_cases.reserve(configs.size()); - for (auto& [config, options] : configs) { - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = - make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - impl_selector, - &options)); - } - - if (impl_selector == "im2col_auto") { - Conv2dConfig config{ - OutInChannels(1, 4), - InputSize2D(91, 91), - KernelSize(91, 91), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}; - Im2colUnsignedTestOptions options; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = - make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - impl_selector, - &options)); - } - - const vkapi::Adapter& adapter = *context->adapter_ptr(); - if (impl_selector == "im2col_unsigned" || - (impl_selector == "im2col_auto" && can_use_unsigned_pw_dot(adapter, 4))) { - const int32_t buffer_output_channels = utils::safe_downcast( - static_cast(adapter.max_texture2d_dim()) * 4 + 1); - Conv2dConfig config{ - OutInChannels(buffer_output_channels, 4), - InputSize2D(1, 1), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}; - Im2colUnsignedTestOptions options; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = - make_test_case_name(config, false, utils::kBuffer, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kBuffer, - utils::kPackedInt8_4W4C, - impl_selector, - &options)); - } - - return test_cases; -} - -// Reference implementation for activation, weight, and output quantized conv2d -static void conv2d_q8ta_q8csw_q8to_reference_impl(TestCase& test_case) { - // Extract input specifications - int32_t idx = 0; - const ValueSpec& input_spec = test_case.inputs()[idx++]; - const ValueSpec& input_scale_spec = test_case.inputs()[idx++]; - const ValueSpec& input_zeros_spec = test_case.inputs()[idx++]; - const ValueSpec& weight_spec = test_case.inputs()[idx++]; - const ValueSpec& weight_sums_spec = test_case.inputs()[idx++]; - (void)weight_sums_spec; - const ValueSpec& weight_scales_spec = test_case.inputs()[idx++]; - const ValueSpec& output_scale_spec = test_case.inputs()[idx++]; - const ValueSpec& output_zeros_spec = test_case.inputs()[idx++]; - const ValueSpec& bias_spec = test_case.inputs()[idx++]; - const ValueSpec& kernel_size_spec = test_case.inputs()[idx++]; - const ValueSpec& stride_spec = test_case.inputs()[idx++]; - const ValueSpec& padding_spec = test_case.inputs()[idx++]; - const ValueSpec& dilation_spec = test_case.inputs()[idx++]; - const ValueSpec& groups_spec = test_case.inputs()[idx++]; - const ValueSpec& activation_spec = test_case.inputs()[idx++]; - const ValueSpec& layout_spec = test_case.inputs()[idx++]; - (void)layout_spec; // Not used in reference implementation - const ValueSpec& output_layout_spec = test_case.inputs()[idx++]; - (void)output_layout_spec; // Not used in reference implementation - const ValueSpec& impl_selector_spec = test_case.inputs()[idx++]; - (void)impl_selector_spec; // Not used in reference implementation - - // Extract output specification (mutable reference) - ValueSpec& output_spec = test_case.outputs()[0]; - - // Get tensor dimensions - auto input_sizes = input_spec.get_tensor_sizes(); // [N, C_in, H_in, W_in] - auto weight_sizes = - weight_spec.get_tensor_sizes(); // [C_out, C_in_per_group * K_h * K_w] - auto output_sizes = - output_spec.get_tensor_sizes(); // [N, C_out, H_out, W_out] - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t H_in = input_sizes[2]; - int64_t W_in = input_sizes[3]; - int64_t C_out = output_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - // Get kernel dimensions from kernel_size ValueSpec - auto kernel_size_data = kernel_size_spec.get_int32_data(); - int64_t K_h = kernel_size_data[0]; - int64_t K_w = kernel_size_data[1]; - - // Get stride, padding, dilation, and groups - auto stride_data = stride_spec.get_int32_data(); - auto padding_data = padding_spec.get_int32_data(); - auto dilation_data = dilation_spec.get_int32_data(); - int64_t stride_h = stride_data[0]; - int64_t stride_w = stride_data[1]; - int64_t pad_h = padding_data[0]; - int64_t pad_w = padding_data[1]; - int64_t dilation_h = dilation_data[0]; - int64_t dilation_w = dilation_data[1]; - int64_t groups = groups_spec.get_int_value(); - - const int64_t reference_operations = - N * C_out * H_out * W_out * (C_in / groups) * K_h * K_w; - const bool has_large_dimension = N > kRefDimSizeLimit || - C_in > kRefDimSizeLimit || H_in > kRefDimSizeLimit || - W_in > kRefDimSizeLimit || C_out > kRefDimSizeLimit; - if (has_large_dimension && reference_operations > kRefOperationLimit) { - throw std::invalid_argument( - "One or more dimensions exceed the allowed limit for reference implementation."); - std::cout - << "Reference implementation: computation may take some time for large tensors..." - << std::endl; - } - - if (input_spec.dtype != vkapi::kFloat) { - throw std::invalid_argument("Unsupported dtype"); - } - - // Get raw data pointers - auto& input_data = input_spec.get_float_data(); - const float input_scale = input_scale_spec.get_float_value(); - const int32_t input_zero_point = input_zeros_spec.get_int_value(); - - auto& weight_data = weight_spec.get_int8_data(); - auto& weight_scales_data = weight_scales_spec.get_float_data(); - auto& bias_data = bias_spec.get_float_data(); - - const float output_scale = output_scale_spec.get_float_value(); - const int32_t output_zero_point = output_zeros_spec.get_int_value(); - - // Calculate channels per group for grouped convolution - int64_t C_in_per_group = C_in / groups; - int64_t C_out_per_group = C_out / groups; - - // Calculate number of output elements - int64_t num_output_elements = N * C_out * H_out * W_out; - - auto& ref_data = output_spec.get_ref_float_data(); - ref_data.resize(num_output_elements); - - const int64_t in_features = utils::align_up_4(C_in_per_group * K_h * K_w); - - // Perform activation, weight, and output quantized conv2d operation - for (int64_t n = 0; n < N; ++n) { - for (int64_t out_c = 0; out_c < C_out; ++out_c) { - for (int64_t out_h = 0; out_h < H_out; ++out_h) { - for (int64_t out_w = 0; out_w < W_out; ++out_w) { - int32_t int_sum = 0; - int32_t weight_sum = 0; // Track weight sum on the fly - - // Determine which group this output channel belongs to - int64_t group_idx = out_c / C_out_per_group; - int64_t in_c_start = group_idx * C_in_per_group; - int64_t in_c_end = (group_idx + 1) * C_in_per_group; - - // Convolution operation with integer accumulation - for (int64_t in_c = in_c_start; in_c < in_c_end; ++in_c) { - for (int64_t kh = 0; kh < K_h; ++kh) { - for (int64_t kw = 0; kw < K_w; ++kw) { - // Calculate input position with dilation - int64_t in_h = out_h * stride_h - pad_h + kh * dilation_h; - int64_t in_w = out_w * stride_w - pad_w + kw * dilation_w; - - // Check bounds (zero padding) - if (in_h >= 0 && in_h < H_in && in_w >= 0 && in_w < W_in) { - // Get input value and quantize to int8 - int64_t input_idx = n * (C_in * H_in * W_in) + - in_c * (H_in * W_in) + in_h * W_in + in_w; - - float quant_input_f = - std::round(input_data[input_idx] / input_scale) + - input_zero_point; - quant_input_f = - std::min(std::max(quant_input_f, -128.0f), 127.0f); - int8_t quantized_input = static_cast(quant_input_f); - - // Get quantized weight (already int8) - // Weight layout: [C_out, C_in_per_group * K_h * K_w] - int64_t weight_idx = out_c * in_features + - (kh * (K_w * C_in_per_group) + kw * C_in_per_group + - (in_c % C_in_per_group)); - int8_t quantized_weight = weight_data[weight_idx]; - - // Integer multiplication and accumulation - int_sum += static_cast(quantized_input) * - static_cast(quantized_weight); - - // Track weight sum for this output channel on the fly - weight_sum += static_cast(quantized_weight); - } else { - // For zero padding, we still need to account for the weight - // in weight_sum when input is effectively 0 (but quantized 0 - // is input_zero_point) - int64_t weight_idx = out_c * in_features + - (kh * (K_w * C_in_per_group) + kw * C_in_per_group + - (in_c % C_in_per_group)); - int8_t quantized_weight = weight_data[weight_idx]; - - // Add contribution from zero-padded input (quantized zero = - // input_zero_point) - int_sum += static_cast(input_zero_point) * - static_cast(quantized_weight); - - // Track weight sum for this output channel on the fly - weight_sum += static_cast(quantized_weight); - } - } - } - } - - // Convert accumulated integer result to float and apply scales - // Final result = (int_sum - zero_point_correction) * input_scale * - // weight_scale + bias zero_point_correction = input_zero_point * - // sum_of_weights_for_this_output_channel - int32_t zero_point_correction = input_zero_point * weight_sum; - int32_t accum_adjusted = int_sum - zero_point_correction; - float float_result = - accum_adjusted * input_scale * weight_scales_data[out_c]; - - if (!bias_spec.is_none()) { - float_result += bias_data[out_c]; - } - if (activation_spec.get_string_value() == "relu") { - float_result = std::max(float_result, 0.0f); - } - - // Quantize the output to int8 - float quant_output_f = - std::round(float_result / output_scale) + output_zero_point; - quant_output_f = std::min(std::max(quant_output_f, -128.0f), 127.0f); - int8_t quantized_output = static_cast(quant_output_f); - - // Dequantize back to float - float dequant_output = - (static_cast(quantized_output) - output_zero_point) * - output_scale; - - int64_t output_idx = n * (C_out * H_out * W_out) + - out_c * (H_out * W_out) + out_h * W_out + out_w; - ref_data[output_idx] = dequant_output; - } - } - } - } -} - -static void reference_impl(TestCase& test_case) { - conv2d_q8ta_q8csw_q8to_reference_impl(test_case); -} - -// The impl selector holds one of "", "general", or "im2col"; activation and -// other string inputs use disjoint values. Overwrite it by value so a -// reordered input list fails loudly instead of mutating the wrong spec. -// Note: for route == "direct" the measured run already forces "general", so -// this reference re-executes the identical implementation and only checks -// determinism; genuine cross-implementation correctness comes from the -// auto/im2col legs. -static void scenex_direct_reference(TestCase& test_case) { - TestCase direct_case = test_case; - bool found_selector = false; - for (auto it = direct_case.inputs().rbegin(); - it != direct_case.inputs().rend(); - ++it) { - if (it->is_string() && - (it->get_string_value().empty() || - it->get_string_value() == "general" || - it->get_string_value() == "im2col")) { - it->string_data = "general"; - found_selector = true; - break; - } - } - if (!found_selector) { - throw std::runtime_error("scenex reference: impl selector input not found"); - } - execute_test_case( - direct_case, - /*warmup_runs=*/1, - /*benchmark_runs=*/1, - /*chained_dispatches=*/1, - /*write_outputs=*/true); - test_case.outputs().at(0).get_ref_float_data() = - direct_case.outputs().at(0).get_float_data(); -} - -// Custom FLOP calculator for quantized conv2d operation -static int64_t quantized_conv2d_flop_calculator(const TestCase& test_case) { - int kernel_idx = 9; // kernel_size is at index 9 for q8ta_q8csw_q8to - - // Get input and weight dimensions - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - - const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t C_out = output_sizes[1]; - int64_t K_h = kernel_sizes[0]; - int64_t K_w = kernel_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - // Calculate FLOPs for quantized conv2d operation - // Each output element requires: - // - C_in * K_h * K_w multiply-accumulate operations - // - Additional operations for quantization/dequantization - int64_t output_elements = N * C_out * H_out * W_out; - int64_t ops_per_output = C_in * K_h * K_w; - - int64_t flop = output_elements * (ops_per_output); - - return flop; -} - -static void execute_streaming_dynamic_shrink_test() { - Conv2dConfig config = { - OutInChannels(4, 64), - InputSize2D(30, 99), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1, - 10}; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = - make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); - TestCase test_case = create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - /*impl_selector=*/"im2col_auto", - /*im2col_options=*/nullptr, - /*input_scale_val=*/1.0f, - /*input_data_gen=*/DataGenType::RANDINT); - for (ValueSpec& input : test_case.inputs()) { - input.ensure_data_generated(/*seed=*/0); - } - - BenchmarkGraph benchmark_graph = setup_compute_graph( - test_case, test_case.operator_name(), /*op_invocations_per_execute=*/1); - ComputeGraph& graph = *benchmark_graph.graph; - graph.prepare(); - graph.prepack(); - - const std::vector upper_input_sizes = test_case.inputs().at(0).sizes; - const std::vector shrunk_input_sizes = {1, 64, 20, 51}; - const std::vector shrunk_output_sizes = {1, 4, 20, 51}; - TestCase shrunk_case = test_case; - shrunk_case.inputs().at(0).sizes = shrunk_input_sizes; - shrunk_case.inputs().at(0).resize_data(1 * 64 * 20 * 51); - shrunk_case.outputs().at(0).sizes = shrunk_output_sizes; - shrunk_case.outputs().at(0).resize_data(1 * 4 * 20 * 51); - reference_impl(shrunk_case); - - constexpr int kRepetitions = 4; - for (int repetition = 0; repetition < kRepetitions; ++repetition) { - graph.resize_input(0, upper_input_sizes); - graph.propagate_resize(); - graph.maybe_cast_and_copy_into_staging( - graph.inputs().at(0).staging, - test_case.inputs().at(0).get_data_ptr(), - test_case.inputs().at(0).numel(), - vkapi::kFloat); - graph.execute(); - - graph.resize_input(0, shrunk_input_sizes); - graph.propagate_resize(); - if (graph.sizes_of(graph.outputs().at(0).value) != shrunk_output_sizes) { - throw std::runtime_error("streaming im2col output did not shrink"); - } - graph.maybe_cast_and_copy_into_staging( - graph.inputs().at(0).staging, - shrunk_case.inputs().at(0).get_data_ptr(), - shrunk_case.inputs().at(0).numel(), - vkapi::kFloat); - graph.execute(); - graph.maybe_cast_and_copy_from_staging( - graph.outputs().at(0).staging, - shrunk_case.outputs().at(0).get_mutable_data_ptr(), - shrunk_case.outputs().at(0).numel(), - vkapi::kFloat); - if (!shrunk_case.outputs().at(0).validate_against_reference( - shrunk_case.get_abs_tolerance(), shrunk_case.get_rel_tolerance())) { - throw std::runtime_error("streaming im2col shrink output was stale"); - } - } - - const Q8taConv2dStreamPlan upper_plan = make_q8ta_conv2d_stream_plan( - /*batch=*/10, - /*flattened_kernel_size=*/576, - /*out_height=*/30, - /*out_width=*/99, - kQ8taConv2dIm2ColScratchBudgetBytes); - if (!upper_plan.feasible || upper_plan.num_tiles != 2 || - upper_plan.rows_per_tile <= 20) { - throw std::runtime_error("dynamic shrink test did not create dead tiles"); - } - const int64_t resized_scratch_bytes = - 576 * upper_plan.rows_per_tile * utils::align_up_4(51); - if (resized_scratch_bytes > kQ8taConv2dIm2ColScratchBudgetBytes) { - throw std::runtime_error("streaming im2col scratch exceeded its cap"); - } - std::cout << "Streaming im2col dynamic shrink PASSED" << std::endl; -} - -// Single usage string for the scenex route-test modes; argument errors -// return 2 like the other CLI errors in main. -static int print_scenex_usage(const char* mode, const char* cases) { - std::cerr << "Usage: " << mode << " <" << cases << ">" - << std::endl; - return 2; -} - -int main(int argc, char* argv[]) { - const vkapi::Adapter& adapter = *vkcompute::api::context()->adapter_ptr(); - const bool prefers_unsigned_dot = - adapter.accelerates_unsigned_packed4x8_dot() && - !adapter.accelerates_signed_packed4x8_dot(); - VK_CHECK_COND( - can_use_unsigned_pw_dot(adapter, kMaxUnsignedDotAccumulatorBytes) == - prefers_unsigned_dot); - VK_CHECK_COND( - !can_use_unsigned_pw_dot(adapter, kMaxUnsignedDotAccumulatorBytes + 1)); - - std::string im2col_impl_selector; - bool narrow_workgroups_only = false; - bool streaming_im2col_only = false; - bool streaming_dynamic_shrink_only = false; - const bool scenex_regular = - argc == 4 && std::string(argv[1]) == "--scenex-regular"; - const bool scenex_grouped = - argc == 4 && std::string(argv[1]) == "--scenex-grouped"; - if (argc >= 2 && std::string(argv[1]) == "--scenex-regular" && - !scenex_regular) { - return print_scenex_usage("--scenex-regular", "0|1"); - } - if (argc >= 2 && std::string(argv[1]) == "--scenex-grouped" && - !scenex_grouped) { - return print_scenex_usage("--scenex-grouped", "0|1|2"); - } - if (!scenex_regular && !scenex_grouped) { - for (int i = 1; i < argc; ++i) { - const std::string arg(argv[i]); - if (arg == "--im2col-path=signed") { - im2col_impl_selector = "im2col"; - } else if (arg == "--im2col-path=unsigned") { - im2col_impl_selector = "im2col_unsigned"; - } else if (arg == "--im2col-path=auto") { - im2col_impl_selector = "im2col_auto"; - } else if (arg == "--narrow-workgroups-only") { - narrow_workgroups_only = true; - } else if (arg == "--streaming-im2col-only") { - streaming_im2col_only = true; - } else if (arg == "--streaming-dynamic-shrink-only") { - streaming_dynamic_shrink_only = true; - } else { - std::cerr << "Unknown argument: " << arg << std::endl; - return 2; - } - } - } - const int selected_modes = static_cast(!im2col_impl_selector.empty()) + - static_cast(narrow_workgroups_only) + - static_cast(streaming_im2col_only) + - static_cast(streaming_dynamic_shrink_only) + - static_cast(scenex_regular) + static_cast(scenex_grouped); - if (selected_modes > 1) { - std::cerr << "Test mode selectors are mutually exclusive" << std::endl; - return 2; - } - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(true); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "Quantized Conv2d Operation with Output Quantization Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - int warmup_runs = 1; - int benchmark_runs = 1; - - if (streaming_dynamic_shrink_only) { - execute_streaming_dynamic_shrink_test(); - return 0; - } -#ifdef DEBUG_MODE - std::function()> test_case_generator = - generate_quantized_conv2d_easy_cases; -#else - std::function()> test_case_generator = - [im2col_impl_selector]() { - return im2col_impl_selector.empty() - ? generate_quantized_conv2d_test_cases() - : generate_im2col_unsigned_test_cases(im2col_impl_selector); - }; -#endif - if (narrow_workgroups_only) { - test_case_generator = generate_narrow_workgroup_test_cases; - } else if (streaming_im2col_only) { - test_case_generator = generate_streaming_im2col_test_cases; -#ifndef DEBUG_MODE - } else if (selected_modes == 0) { - // The default run also covers the unified tile path: multi-tile - // dispatches, grouped/streaming shapes, and the fallback kernel. - test_case_generator = [base_generator = test_case_generator]() { - auto cases = base_generator(); - const auto streaming_cases = generate_streaming_im2col_test_cases(); - cases.insert(cases.end(), streaming_cases.begin(), streaming_cases.end()); - return cases; - }; -#endif - } else if (scenex_regular) { - const std::string route = argv[2]; - const std::string case_arg = argv[3]; - if ((route != "auto" && route != "direct" && route != "im2col") || - (case_arg != "0" && case_arg != "1")) { - return print_scenex_usage("--scenex-regular", "0|1"); - } - const int case_index = case_arg == "0" ? 0 : 1; - test_case_generator = [route, case_index]() { - return std::vector{ - generate_scenex_regular_test_case(route, case_index)}; - }; - ref_fn = scenex_direct_reference; - warmup_runs = 3; - benchmark_runs = 10; - } else if (scenex_grouped) { - const std::string route = argv[2]; - const std::string case_arg = argv[3]; - if ((route != "auto" && route != "direct" && route != "im2col") || - (case_arg != "0" && case_arg != "1" && case_arg != "2")) { - return print_scenex_usage("--scenex-grouped", "0|1|2"); - } - const int case_index = case_arg == "0" ? 0 : case_arg == "1" ? 1 : 2; - test_case_generator = [route, case_index]() { - return std::vector{ - generate_scenex_grouped_test_case(route, case_index)}; - }; - ref_fn = scenex_direct_reference; - warmup_runs = 3; - benchmark_runs = 10; - } - - auto results = execute_test_cases( - test_case_generator, - quantized_conv2d_flop_calculator, - "QuantizedConv2dQ8ToQ8To", - warmup_runs, - benchmark_runs, - ref_fn); - -#ifndef DEBUG_MODE - if (selected_modes == 0) { - execute_streaming_dynamic_shrink_test(); - } -#endif - - return 0; -} diff --git a/backends/vulkan/test/custom_ops/test_q8ta_qdq.cpp b/backends/vulkan/test/custom_ops/test_q8ta_qdq.cpp deleted file mode 100644 index af48018d31a..00000000000 --- a/backends/vulkan/test/custom_ops/test_q8ta_qdq.cpp +++ /dev/null @@ -1,381 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include -#include -#include -#include -#include -#include "utils.h" - -#include - -// #define DEBUG_MODE - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 512; - -// Configuration struct for tensor quantize-dequantize testing -struct QDQ8BitConfig { - std::vector shape; // Tensor shape (can be any dimensionality) - std::string test_case_name = "placeholder"; - std::string op_name = "q_dq_8bit_per_tensor"; -}; - -// Utility function to create a test case from a QDQ8BitConfig -TestCase create_test_case_from_config( - const QDQ8BitConfig& config, - utils::StorageType storage_type, - vkapi::ScalarType input_dtype, - utils::GPUMemoryLayout fp_memory_layout, - utils::GPUMemoryLayout quantized_memory_layout, - const std::string& impl_selector = "") { - TestCase test_case; - - // Create a descriptive name for the test case - // QDQ: fp input -> int8 staging -> fp output (label as f32->f32) - std::string prefix = config.test_case_name; // "ACCU" or "PERF" - std::string storage_str = repr_str(storage_type, fp_memory_layout) + "->" + - repr_str(utils::kBuffer, quantized_memory_layout); - std::string suffix = impl_selector.empty() ? "" : "[" + impl_selector + "]"; - std::string test_name = make_test_label( - prefix, - dtype_short(input_dtype), - dtype_short(input_dtype), - shape_bracket(config.shape), - storage_str, - suffix); - test_case.set_name(test_name); - - // Set the operator name for the test case - std::string operator_name = "test_etvk." + config.op_name + ".default"; - test_case.set_operator_name(operator_name); - - // Input tensor (float) - any dimensionality - ValueSpec input_tensor( - config.shape, - input_dtype, - storage_type, - fp_memory_layout, - DataGenType::RANDOM); - - float scale_val = 0.007112; - ValueSpec scale(scale_val); - - // Generate random zero point within quantization range - int32_t zero_point_val = 0; - ValueSpec zero_point(zero_point_val); - - // GPUMemoryLayout as integer (will be cast in the operator) - int32_t layout_int = static_cast(quantized_memory_layout); - ValueSpec layout_spec(layout_int); - - // impl_selector string - ValueSpec impl_selector_spec = ValueSpec::make_string(impl_selector); - - // Output tensor (float) - same shape as input - ValueSpec output_tensor( - config.shape, - input_dtype, - storage_type, - fp_memory_layout, - DataGenType::ZEROS); - - // Add all specs to test case - test_case.add_input_spec(input_tensor); - test_case.add_input_spec(scale); - test_case.add_input_spec(zero_point); - test_case.add_input_spec(layout_spec); - test_case.add_input_spec(impl_selector_spec); - test_case.add_output_spec(output_tensor); - - test_case.set_abs_tolerance(scale_val + 1e-4); - - // Use layout-only filter for this test since quantize/dequantize ARE the - // operations being tested, not overhead - test_case.set_shader_filter(kLayoutOnlyShaderFilter); - - return test_case; -} - -// Generate easy test cases for q_dq_8bit operation (for debugging) -std::vector generate_q_dq_8bit_easy_cases() { - std::vector test_cases; - - std::vector> shapes = { - {1, 16, 16, 16}, // 4D: [N, C, H, W] - {1, 144}, // 2D: exercises block config with ndim < 4 - {1, 90}, // 2D: matches skin_seg model's keypoint/bbox tensor sizes - {2, 1, 3, 16, 16}, // 5D: exercises high-dim batch support - {2, 1, 1, 4, 8, 16}, // 6D: exercises high-dim batch support - }; - - // FP memory layouts to test - std::vector fp_layouts = { - utils::kWidthPacked, - utils::kChannelsPacked, - }; - - // Quantized memory layouts to test - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - std::vector storage_types = {utils::kBuffer}; - std::vector float_types = {vkapi::kFloat}; - - // Generate test cases for each combination - for (const auto& shape : shapes) { - QDQ8BitConfig config = {shape, "ACCU"}; - for (const auto& fp_layout : fp_layouts) { - for (const auto& quant_layout : quant_layouts) { - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - test_cases.push_back(create_test_case_from_config( - config, storage_type, input_dtype, fp_layout, quant_layout)); - // For 4W4C layout, also test with legacy implementation - if (quant_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - storage_type, - input_dtype, - fp_layout, - quant_layout, - /*impl_selector=*/"legacy_4w4c")); - } - } - } - } - } - } - - return test_cases; -} - -// Generate test cases for q_dq_8bit operation -std::vector generate_q_dq_8bit_test_cases() { - std::vector test_cases; - - // Shapes to test (no layout specified - will be combined with all layouts) - std::vector> shapes = { - // 1D tensors - {144}, - {90}, - - // 2D tensors (exercises block config with ndim < 4) - {1, 144}, - {1, 90}, - {1, 4}, - {3, 32}, - - // 3D tensors - {1, 16, 32}, - {1, 3, 64}, - - // Small test cases for correctness - {1, 3, 16, 16}, - {1, 8, 32, 32}, - {1, 16, 24, 24}, - {1, 32, 12, 12}, - {1, 1, 64, 64}, - {1, 3, 64, 64}, - {1, 4, 16, 16}, - - // Different tensor sizes - {1, 8, 20, 20}, - {1, 16, 14, 14}, - {1, 8, 28, 28}, - - // Odd tensor sizes - {1, 3, 15, 15}, - {1, 13, 31, 31}, - {1, 17, 23, 23}, - - // Performance test cases (larger tensors) - {1, 64, 128, 128}, - {1, 32, 64, 64}, - {1, 128, 56, 56}, - - // 5D tensors (high-dim batch support) - {2, 1, 3, 16, 16}, - {1, 2, 8, 8, 8}, - - // 6D tensors (high-dim batch support) - {2, 1, 1, 4, 8, 16}, - {1, 1, 2, 3, 8, 8}, - }; - - // FP memory layouts to test - std::vector fp_layouts = { - utils::kWidthPacked, - utils::kChannelsPacked, - }; - - // Quantized memory layouts to test - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - // Test with buffer storage only - the unified block-based shaders only - // support buffer-backed floating-point tensors. Texture storage is tested - // separately by qdq8ta_conv2d_activations which uses the layout-specific - // shaders. - std::vector storage_types = { - utils::kBuffer, utils::kTexture3D}; - - // Generate all combinations - for (const auto& shape : shapes) { - // Generate test case name prefix from shape dimensions - std::string prefix = "ACCU"; - for (const auto& dim : shape) { - if (dim > kRefDimSizeLimit) { - prefix = "PERF"; - break; - } - } - - // Skip texture3d for high-dim tensors (textures are limited to 4D) - const bool is_highdim = shape.size() > 4; - const auto& effective_storage_types = is_highdim - ? std::vector{utils::kBuffer} - : storage_types; - - for (const auto& fp_layout : fp_layouts) { - for (const auto& quant_layout : quant_layouts) { - for (const auto& storage_type : effective_storage_types) { - QDQ8BitConfig config; - config.shape = shape; - config.test_case_name = prefix; - - test_cases.push_back(create_test_case_from_config( - config, storage_type, vkapi::kFloat, fp_layout, quant_layout)); - // For 4W4C layout, also test with legacy implementation - // (legacy path doesn't support high-dim tensors) - if (!is_highdim && fp_layout == utils::kChannelsPacked && - quant_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - storage_type, - vkapi::kFloat, - fp_layout, - quant_layout, - /*impl_selector=*/"legacy_4w4c")); - } - } - } - } - } - - return test_cases; -} - -// Reference implementation for q_dq_8bit operation -void q_dq_8bit_reference_impl(TestCase& test_case) { - int32_t idx = 0; - const ValueSpec& input_spec = test_case.inputs()[idx++]; - const ValueSpec& scale_spec = test_case.inputs()[idx++]; - const ValueSpec& zero_point_spec = test_case.inputs()[idx++]; - const ValueSpec& layout_spec = test_case.inputs()[idx++]; - (void)layout_spec; // Not used in reference implementation - - // Extract output specification - ValueSpec& output_spec = test_case.outputs()[0]; - - // Get tensor dimensions (arbitrary dimensionality) - auto input_sizes = input_spec.get_tensor_sizes(); - - // Calculate total number of elements - int64_t num_elements = 1; - for (const auto& dim : input_sizes) { - num_elements *= dim; - } - - // Skip for large tensors since computation time will be extremely slow - for (const auto& dim : input_sizes) { - if (dim > kRefDimSizeLimit) { - throw std::invalid_argument( - "One or more dimensions exceed the allowed limit for reference implementation."); - } - } - - if (input_spec.dtype != vkapi::kFloat) { - throw std::invalid_argument("Unsupported dtype"); - } - - // Get raw data pointers - auto& input_data = input_spec.get_float_data(); - - // Extract the randomized scale and zero point values - float scale = scale_spec.get_float_value(); - int32_t zero_point = zero_point_spec.get_int_value(); - int32_t quant_min = -128; - int32_t quant_max = 127; - - // Prepare output data - auto& ref_data = output_spec.get_ref_float_data(); - ref_data.resize(num_elements); - - // Perform quantize-dequantize operation on each element - for (int64_t i = 0; i < num_elements; ++i) { - float input_val = input_data[i]; - - // Quantize: quantized = round(input / scale + zero_point) - float quantized_float = std::round(input_val / scale) + zero_point; - - // Clamp to quantization range - quantized_float = std::max(quantized_float, static_cast(quant_min)); - quantized_float = std::min(quantized_float, static_cast(quant_max)); - - int32_t quantized_int = static_cast(quantized_float); - - // Dequantize: output = (quantized - zero_point) * scale - float dequantized = (quantized_int - zero_point) * scale; - - ref_data[i] = dequantized; - } -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(false); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Q/DQ 8-bit Per Tensor Operation Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = q_dq_8bit_reference_impl; - - auto results = execute_test_cases( -#ifdef DEBUG_MODE - generate_q_dq_8bit_easy_cases, -#else - generate_q_dq_8bit_test_cases, -#endif - "QDQ8Bit", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/custom_ops/test_sdpa.cpp b/backends/vulkan/test/custom_ops/test_sdpa.cpp deleted file mode 100644 index 55d2173e3cd..00000000000 --- a/backends/vulkan/test/custom_ops/test_sdpa.cpp +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include -#include -#include -#include -#include - -#include -#include - -#include "utils.h" - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -// Correctness is only checked for these small shapes; larger perf shapes throw -// std::invalid_argument from the reference (framework marks them SKIPPED) to -// avoid an O(H*S*context*D) CPU reference on the large perf matrix. -static constexpr int64_t kRefContextLenLimit = 256; - -// When true (env SDPA_NO_CHAIN=1), each graph.execute() dispatches the SDPA op -// exactly once (op_invocations_per_execute=1), disabling the framework's -// probe-then-scale chaining. Chaining stacks many back-to-back QK/softmax/AV -// triples in one command buffer; consecutive triples pipeline on the GPU so -// their timestamp windows overlap, which inflates/misattributes per-dispatch -// durations (notably the AV dispatch) at the large chained_dispatches factors -// picked for the cheap small-context decode cases. One-invocation-per-execute -// removes the inter-invocation overlap; stability comes from warmup + a large -// median-of-N instead. -static bool no_chain_mode() { - const char* v = std::getenv("SDPA_NO_CHAIN"); - return v != nullptr && v[0] == '1'; -} - -// When true (env SDPA_DECODE_ONLY=1), generate only the 9 decode cases. -static bool decode_only_mode() { - const char* v = std::getenv("SDPA_DECODE_ONLY"); - return v != nullptr && v[0] == '1'; -} - -// LLM SDPA (llama.custom_sdpa) shape: -// q: [1, S, n_heads, head_dim] (DHSB, width-packed) -// k/v cache:[1, context_len, n_kv_heads, head_dim] -struct SDPAConfig { - int64_t head_dim; - int64_t n_heads; - int64_t n_kv_heads; - int64_t seq_len; // S: query tokens (1 for decode, >1 for prefill) - int64_t context_len; // total KV length (kv_len) - std::string model; // label only - std::string regime; // "decode" / "prefill", label only -}; - -static std::vector as_float_data(const ValueSpec& spec) { - if (spec.dtype == vkapi::kFloat) { - return spec.get_float_data(); - } - if (spec.dtype == vkapi::kHalf) { - const auto& half_bits = spec.get_half_data(); - std::vector out(half_bits.size()); - for (size_t i = 0; i < half_bits.size(); ++i) { - out[i] = half_to_float(half_bits[i]); - } - return out; - } - throw std::invalid_argument("as_float_data: unsupported dtype"); -} - -static TestCase create_sdpa_test_case( - const SDPAConfig& config, - vkapi::ScalarType dtype, - utils::StorageType storage_type, - const std::string& impl) { - TestCase test_case; - - const bool is_perf = config.context_len > kRefContextLenLimit; - const std::string prefix = is_perf ? "PERF" : "ACCU"; - const std::string storage_str = repr_str(storage_type, utils::kWidthPacked); - const std::string dtype_str = dtype_short(dtype); - - const std::string shape = "D" + std::to_string(config.head_dim) + " H" + - std::to_string(config.n_heads) + " Hkv" + - std::to_string(config.n_kv_heads) + " S" + - std::to_string(config.seq_len) + " C" + - std::to_string(config.context_len); - - const std::string suffix = - "[" + config.model + " " + config.regime + " " + impl + "]"; - - test_case.set_name(make_test_label( - prefix, dtype_str, dtype_str, shape, storage_str, suffix)); - test_case.set_operator_name("test_etvk.test_sdpa.default"); - - // q: [1, S, n_heads, head_dim] - ValueSpec q( - {1, config.seq_len, config.n_heads, config.head_dim}, - dtype, - storage_type, - utils::kWidthPacked, - DataGenType::RANDOM); - - // k_cache / v_cache: [1, context_len, n_kv_heads, head_dim] - ValueSpec k_cache( - {1, config.context_len, config.n_kv_heads, config.head_dim}, - dtype, - storage_type, - utils::kWidthPacked, - DataGenType::RANDOM); - ValueSpec v_cache( - {1, config.context_len, config.n_kv_heads, config.head_dim}, - dtype, - storage_type, - utils::kWidthPacked, - DataGenType::RANDOM); - - ValueSpec impl_selector = ValueSpec::make_string(impl); - - // out: [1, S, n_heads, head_dim] - ValueSpec output( - {1, config.seq_len, config.n_heads, config.head_dim}, - dtype, - storage_type, - utils::kWidthPacked, - DataGenType::ZEROS); - - test_case.add_input_spec(q); - test_case.add_input_spec(k_cache); - test_case.add_input_spec(v_cache); - test_case.add_input_spec(impl_selector); - test_case.add_output_spec(output); - - if (no_chain_mode()) { - test_case.set_op_invocations_per_execute(1); - } - - if (dtype == vkapi::kHalf) { - test_case.set_abs_tolerance(1e-2f); - test_case.set_rel_tolerance(1e-2f); - } else { - test_case.set_abs_tolerance(1e-3f); - test_case.set_rel_tolerance(1e-3f); - } - - return test_case; -} - -// Reference: causal SDPA over the KV cache. -// q:[1,S,H,D], k/v cache:[1,C,Hkv,D], input_pos = C - S. -// For query row s (absolute position input_pos + s), attends to cache -// positions [0, input_pos + s]. GQA: head h maps to kv head h / (H/Hkv). -static void sdpa_reference_impl(TestCase& test_case) { - const auto& q = test_case.inputs()[0]; - const auto& k = test_case.inputs()[1]; - const auto& v = test_case.inputs()[2]; - - const auto q_sizes = q.get_tensor_sizes(); - const auto k_sizes = k.get_tensor_sizes(); - - const int64_t S = q_sizes[1]; - const int64_t H = q_sizes[2]; - const int64_t D = q_sizes[3]; - const int64_t C = k_sizes[1]; - const int64_t Hkv = k_sizes[2]; - - if (C > kRefContextLenLimit) { - throw std::invalid_argument("sdpa reference: perf shape, skipping"); - } - - const int64_t input_pos = C - S; - const int64_t heads_per_kv = H / Hkv; - const float scale = 1.0f / std::sqrt(static_cast(D)); - - const auto q_data = as_float_data(q); - const auto k_data = as_float_data(k); - const auto v_data = as_float_data(v); - - ValueSpec& output = test_case.outputs()[0]; - auto& ref = output.get_ref_float_data(); - ref.assign(S * H * D, 0.0f); - - // Index helpers (contiguous WHCN-flattened as [1, dim1, dim2, dim3]). - auto q_idx = [&](int64_t s, int64_t h, int64_t d) { - return (s * H + h) * D + d; - }; - auto kv_idx = [&](int64_t c, int64_t hk, int64_t d) { - return (c * Hkv + hk) * D + d; - }; - - std::vector scores(C); - for (int64_t s = 0; s < S; ++s) { - const int64_t attend_len = input_pos + s + 1; // causal - for (int64_t h = 0; h < H; ++h) { - const int64_t hk = h / heads_per_kv; - - float max_score = -std::numeric_limits::infinity(); - for (int64_t c = 0; c < attend_len; ++c) { - float dot = 0.0f; - for (int64_t d = 0; d < D; ++d) { - dot += q_data[q_idx(s, h, d)] * k_data[kv_idx(c, hk, d)]; - } - dot *= scale; - scores[c] = dot; - max_score = std::max(max_score, dot); - } - - float denom = 0.0f; - for (int64_t c = 0; c < attend_len; ++c) { - scores[c] = std::exp(scores[c] - max_score); - denom += scores[c]; - } - - for (int64_t d = 0; d < D; ++d) { - float acc = 0.0f; - for (int64_t c = 0; c < attend_len; ++c) { - acc += scores[c] * v_data[kv_idx(c, hk, d)]; - } - ref[q_idx(s, h, d)] = acc / denom; - } - } - } -} - -// FLOPs: QK (2*S*C*D) + AV (2*S*C*D) per head, summed over heads. Softmax -// is negligible. Uses the causal-average context (~C/2) is ignored; report -// full-C dense FLOPs as an upper bound proxy. -static int64_t sdpa_flop_calculator(const TestCase& test_case) { - const auto q_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto k_sizes = test_case.inputs()[1].get_tensor_sizes(); - const int64_t S = q_sizes[1]; - const int64_t H = q_sizes[2]; - const int64_t D = q_sizes[3]; - const int64_t C = k_sizes[1]; - return 4 * H * S * C * D; -} - -static std::vector generate_sdpa_test_cases() { - std::vector test_cases; - - struct ModelDims { - std::string name; - int64_t head_dim; - int64_t n_heads; - int64_t n_kv_heads; - }; - - const std::vector models = { - {"Llama-3.2-1B", 64, 32, 8}, - {"Qwen3-0.6B", 128, 16, 8}, - {"Phi-4-mini", 128, 24, 8}, - }; - - // Decode: S=1, sweep context_len. - const std::vector decode_context_lens = {512, 1024, 4096}; - // Prefill: S == context_len. - const std::vector prefill_seq_lens = {128, 512}; - - // Perf runs use fp16 texture (matches the LLM decode/prefill production - // path). A couple of small ACCU shapes validate correctness in fp32. - const auto dtype = vkapi::kHalf; - const auto storage = utils::kTexture3D; - - // Decode (S==1) picks a coop AV shader; exercise both the GQA-reuse variant - // and the per-query-head variant for every decode case. Prefill (tiled) is - // unaffected by the selector, so it runs a single case. - const std::vector decode_impls = {"gqa", "non_gqa"}; - - for (const auto& m : models) { - for (int64_t c : decode_context_lens) { - SDPAConfig cfg; - cfg.head_dim = m.head_dim; - cfg.n_heads = m.n_heads; - cfg.n_kv_heads = m.n_kv_heads; - cfg.seq_len = 1; - cfg.context_len = c; - cfg.model = m.name; - cfg.regime = "decode"; - for (const auto& impl : decode_impls) { - test_cases.push_back(create_sdpa_test_case(cfg, dtype, storage, impl)); - } - } - if (decode_only_mode()) { - continue; - } - for (int64_t s : prefill_seq_lens) { - SDPAConfig cfg; - cfg.head_dim = m.head_dim; - cfg.n_heads = m.n_heads; - cfg.n_kv_heads = m.n_kv_heads; - cfg.seq_len = s; - cfg.context_len = s; - cfg.model = m.name; - cfg.regime = "prefill"; - test_cases.push_back( - create_sdpa_test_case(cfg, dtype, storage, "default")); - } - } - - if (decode_only_mode()) { - return test_cases; - } - - // Small ACCU correctness cases (fp32), decode + prefill. Texture is the - // production LLM path; buffer is also validated for decode to guard the - // attn_weights S/context alignment (a decode-shaped buffer allocation has no - // headroom for the shaders' align_up_4 stride unless padded — see sdpa_impl). - { - // Cover D=64 (D4=16) and D=128 (D4=32) with the vendor-default GQA and the - // per-query-head shaders, across texture + buffer. - const std::vector decs = { - {64, 8, 2, 1, 32, "accu", "decode"}, - {128, 8, 2, 1, 32, "accu_d128", "decode"}, - }; - for (const auto& dec : decs) { - for (const auto& storage : {utils::kTexture3D, utils::kBuffer}) { - for (const auto& impl : decode_impls) { - test_cases.push_back( - create_sdpa_test_case(dec, vkapi::kFloat, storage, impl)); - } - } - } - - // Force the head_dim output-tiled GQA variant (Adreno-only in production) - // so its wg x-collapse and the partial_n_tile tail get deterministic - // coverage on any device: D=64/128 give even D4 (fast path); D=4 gives D4=1 - // (odd), exercising the partial-tile checked load. - const std::vector tile2_decs = { - {64, 8, 2, 1, 32, "accu_tile2", "decode"}, - {128, 8, 2, 1, 32, "accu_tile2_d128", "decode"}, - {4, 8, 2, 1, 32, "accu_tile2_d4", "decode"}, - }; - for (const auto& dec : tile2_decs) { - for (const auto& storage : {utils::kTexture3D, utils::kBuffer}) { - test_cases.push_back( - create_sdpa_test_case(dec, vkapi::kFloat, storage, "gqa_tile2")); - } - } - - SDPAConfig pre{64, 8, 2, 16, 16, "accu", "prefill"}; - test_cases.push_back(create_sdpa_test_case( - pre, vkapi::kFloat, utils::kTexture3D, "default")); - } - - return test_cases; -} - -int main(int argc, char* argv[]) { - (void)argc; - (void)argv; - - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "SDPA (llama.custom_sdpa) Benchmark" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = sdpa_reference_impl; - - const bool decode_only = decode_only_mode(); - const int warmup_runs = decode_only ? 10 : 3; - const int benchmark_runs = decode_only ? 30 : 10; - - auto results = execute_test_cases( - generate_sdpa_test_cases, - sdpa_flop_calculator, - "SDPA", - warmup_runs, - benchmark_runs, - ref_fn); - - return 0; -} diff --git a/backends/vulkan/test/custom_ops/utils.cpp b/backends/vulkan/test/custom_ops/utils.cpp deleted file mode 100644 index 066996de262..00000000000 --- a/backends/vulkan/test/custom_ops/utils.cpp +++ /dev/null @@ -1,2482 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include "utils.h" -#include -#include -#include -#include -#include - -#include - -namespace executorch { -namespace vulkan { -namespace prototyping { - -namespace { - -// Upper bound on the auto-sized chained_dispatches factor. Caps the factor -// when an op is fast enough that target_us / probe_us would otherwise -// exceed this; past this point, stacking more dispatches doesn't materially -// improve measurement precision. -constexpr int kMaxChainedDispatches = 200; - -// Floor for probe_us to avoid division-by-zero / a runaway chained_dispatches -// factor when an op finishes faster than the wall clock can resolve. -constexpr float kMinProbeTimeUs = 1.0f; - -} // namespace - -// Benchmark graphs are immutable after input upload, so the operator nodes can -// be recorded repeatedly. Staging uploads/downloads are encoded once: -// repeating them would redo host-visible copies on every repetition and bias -// per-invocation timings. -RepeatedGraphExecutor::RepeatedGraphExecutor( - ComputeGraph& graph, - int repetitions, - OpNodeRange op_nodes) - : graph_(graph) { - VK_CHECK_COND(repetitions > 0); - VK_CHECK_COND(op_nodes.begin <= op_nodes.end); - VK_CHECK_COND(op_nodes.end <= graph_.execute_nodes().size()); - - api::Context* const context = graph_.context(); - context->flush(); - context->set_cmd(/*reusable=*/true); - context->cmd_reset_querypool(); - - for (size_t i = 0; i < op_nodes.begin; ++i) { - graph_.execute_nodes()[i]->encode(&graph_); - } - for (int i = 0; i < repetitions; ++i) { - for (size_t j = op_nodes.begin; j < op_nodes.end; ++j) { - graph_.execute_nodes()[j]->encode(&graph_); - } - } - for (size_t i = op_nodes.end; i < graph_.execute_nodes().size(); ++i) { - graph_.execute_nodes()[i]->encode(&graph_); - } - - command_ = - std::make_unique(std::move(context->extract_cmd())); -} - -void RepeatedGraphExecutor::execute() { - api::Context* const context = graph_.context(); - command_->end(); - - // Intentionally bypasses Context::submit_cmd_to_gpu(): its submit-count - // bookkeeping and threshold-split path only serve submit_compute_job - // recording, which benchmarks don't use. - vkapi::VulkanFence fence = context->fences().get_fence(); - context->adapter_ptr()->submit_cmd( - context->queue(), - command_->get_submit_handle(/*final_use=*/false), - fence.get_submit_handle()); - fence.wait(); - context->fences().return_fence(fence); -} - -int get_seed() { - static int seed = 42; - return seed++; -} - -int get_seed_or_explicit(int explicit_seed) { - if (explicit_seed >= 0) { - return explicit_seed; - } - return get_seed(); -} - -// Forward declarations for data generation utilities -void generate_random_float_data( - std::vector& data, - float min_val = -1.0f, - float max_val = 1.0f, - int explicit_seed = -1); -void generate_random_int_data( - std::vector& data, - int min_val = -10, - int max_val = 10, - int explicit_seed = -1); -void generate_randint_float_data( - std::vector& data, - int min_val = -10, - int max_val = 10, - int explicit_seed = -1); -void generate_randint_half_data( - std::vector& data, - int min_val = -10, - int max_val = 10, - int explicit_seed = -1); -void generate_random_int8_data( - std::vector& data, - int8_t min_val = -10, - int8_t max_val = 10, - int explicit_seed = -1); -void generate_random_uint8_data( - std::vector& data, - uint8_t min_val = 0, - uint8_t max_val = 255, - int explicit_seed = -1); -void generate_random_2xint4_data( - std::vector& data, - int explicit_seed = -1); -void generate_random_2xint4_data( - std::vector& data, - int explicit_seed = -1); -void generate_random_int4_data( - std::vector& data, - int8_t min_val = -8, - int8_t max_val = 7, - int explicit_seed = -1); -void generate_ones_data(std::vector& data); -void generate_zeros_data(std::vector& data); - -// Convert a float32 value to IEEE 754 half-precision (uint16_t) -uint16_t float_to_half(float value) { - uint32_t float_bits; - std::memcpy(&float_bits, &value, sizeof(float)); - uint32_t sign = (float_bits >> 31) & 0x1; - int32_t exponent = static_cast((float_bits >> 23) & 0xFF) - 127; - uint32_t mantissa = float_bits & 0x7FFFFF; - - uint16_t half_val; - if (exponent > 15) { - half_val = static_cast((sign << 15) | 0x7C00); // Inf - } else if (exponent < -14) { - half_val = static_cast(sign << 15); // Zero / subnormal - } else { - half_val = static_cast( - (sign << 15) | (static_cast(exponent + 15) << 10) | - (mantissa >> 13)); - } - return half_val; -} - -// Convert a IEEE 754 half-precision (uint16_t) value to float32 -float half_to_float(uint16_t half_val) { - uint32_t sign = (half_val >> 15) & 0x1; - uint32_t exponent = (half_val >> 10) & 0x1F; - uint32_t mantissa = half_val & 0x3FF; - - float result; - if (exponent == 0) { - result = std::ldexp(static_cast(mantissa), -24); - } else if (exponent == 31) { - result = mantissa ? std::numeric_limits::quiet_NaN() - : std::numeric_limits::infinity(); - } else { - result = std::ldexp( - 1.0f + static_cast(mantissa) / 1024.0f, exponent - 15); - } - if (sign) { - result = -result; - } - return result; -} - -// Output and latency printing utilities -namespace { -static int print_output_enabled = 0; -static int print_latencies_enabled = 0; -static int gpu_timestamps_enabled = 0; -static int debugging_enabled = 0; -} // namespace - -bool print_output() { - return print_output_enabled > 0; -} - -void set_print_output(bool print_output) { - print_output_enabled = print_output ? 1 : 0; -} - -bool print_latencies() { - return print_latencies_enabled > 0; -} - -void set_print_latencies(bool print_latencies) { - print_latencies_enabled = print_latencies ? 1 : 0; -} - -bool use_gpu_timestamps() { - return gpu_timestamps_enabled > 0; -} - -void set_use_gpu_timestamps(bool use_timestamps) { - gpu_timestamps_enabled = use_timestamps ? 1 : 0; -} - -bool debugging() { - return debugging_enabled > 0; -} - -void set_debugging(bool enable_debugging) { - debugging_enabled = enable_debugging ? 1 : 0; -} - -// ValueSpec implementation -void ValueSpec::ensure_unique_data() const { - if (data_.use_count() != 1) { - data_ = std::make_shared(*data_); - } -} - -void ValueSpec::ensure_unique_reference_data() const { - if (reference_data_.use_count() != 1) { - reference_data_ = std::make_shared(*reference_data_); - } -} - -void ValueSpec::generate_tensor_data(int seed) const { - if (spec_type != SpecType::Tensor) { - return; - } - - ensure_unique_data(); - auto& float_data = data_->float_data; - auto& int32_data = data_->int32_data; - auto& half_data = data_->half_data; - auto& int8_data = data_->int8_data; - auto& uint8_data = data_->uint8_data; - - int64_t num_elements = numel(); - - switch (dtype) { - case vkapi::kFloat: { - float_data.resize(num_elements); - if (data_gen_type == DataGenType::RANDOM) { - generate_random_float_data(float_data, -1.0f, 1.0f, seed); - } else if (data_gen_type == DataGenType::RANDOM_SCALES) { - generate_random_float_data(float_data, 0.005, 0.015, seed); - } else if (data_gen_type == DataGenType::RANDINT) { - generate_randint_float_data(float_data, -10, 10, seed); - } else if (data_gen_type == DataGenType::RANDINT8) { - generate_randint_float_data(float_data, -128, 127, seed); - } else if (data_gen_type == DataGenType::RANDINT4) { - generate_randint_float_data(float_data, -8, 7, seed); - } else if (data_gen_type == DataGenType::ONES) { - generate_ones_data(float_data); - } else if (data_gen_type == DataGenType::ZEROS) { - generate_zeros_data(float_data); - } else { - generate_zeros_data(float_data); - } - break; - } - case vkapi::kHalf: { - half_data.resize(num_elements); - if (data_gen_type == DataGenType::RANDOM) { - // Generate random float data first, then convert to IEEE 754 half. - std::vector temp_data(num_elements); - generate_random_float_data(temp_data, -1.0f, 1.0f, seed); - for (size_t i = 0; i < temp_data.size(); ++i) { - half_data[i] = float_to_half(temp_data[i]); - } - } else if (data_gen_type == DataGenType::RANDOM_SCALES) { - // Generate random scales in float, then convert to proper fp16 - std::vector temp_data(num_elements); - generate_random_float_data(temp_data, 0.005f, 0.015f, seed); - for (size_t i = 0; i < temp_data.size(); ++i) { - half_data[i] = float_to_half(temp_data[i]); - } - } else if (data_gen_type == DataGenType::RANDINT) { - generate_randint_half_data(half_data, -10, 10, seed); - } else if (data_gen_type == DataGenType::RANDINT8) { - generate_randint_half_data(half_data, -128, 127, seed); - } else if (data_gen_type == DataGenType::RANDINT4) { - generate_randint_half_data(half_data, -8, 7, seed); - } else if (data_gen_type == DataGenType::ONES) { - std::fill(half_data.begin(), half_data.end(), float_to_half(1.0f)); - } else if (data_gen_type == DataGenType::ZEROS) { - std::fill( - half_data.begin(), - half_data.end(), - static_cast(0)); // 0.0 in half - } else { - std::fill( - half_data.begin(), - half_data.end(), - static_cast(0)); // 0.0 in half - } - break; - } - case vkapi::kInt: { - int32_data.resize(num_elements); - if (data_gen_type == DataGenType::RANDOM) { - generate_random_int_data(int32_data, -10, 10, seed); - } else if (data_gen_type == DataGenType::RANDINT) { - generate_random_int_data( - int32_data, - -10, - 10, - seed); // For int type, RANDINT is same as RANDOM - } else if (data_gen_type == DataGenType::RANDINT8) { - generate_random_int_data(int32_data, -128, 127, seed); - } else if (data_gen_type == DataGenType::RANDINT4) { - generate_random_int_data(int32_data, -8, 7, seed); - } else if (data_gen_type == DataGenType::ONES) { - std::fill(int32_data.begin(), int32_data.end(), 1); - } else if (data_gen_type == DataGenType::ZEROS) { - std::fill(int32_data.begin(), int32_data.end(), 0); - } else { - std::fill(int32_data.begin(), int32_data.end(), 0); - } - break; - } - case vkapi::kChar: { - int8_data.resize(num_elements); - if (data_gen_type == DataGenType::RANDOM) { - generate_random_int8_data(int8_data, -10, 10, seed); - } else if (data_gen_type == DataGenType::RANDINT) { - generate_random_int8_data(int8_data, -10, 10, seed); - } else if (data_gen_type == DataGenType::RANDINT8) { - generate_random_int8_data(int8_data, -128, 127, seed); - } else if (data_gen_type == DataGenType::RANDINT4) { - generate_random_2xint4_data(int8_data, seed); - } else if (data_gen_type == DataGenType::ONES) { - std::fill(int8_data.begin(), int8_data.end(), 1); - } else if (data_gen_type == DataGenType::ONES_INT4) { - int8_t packed_data = (1 << 4) | 1; - std::fill(int8_data.begin(), int8_data.end(), packed_data); - } else if (data_gen_type == DataGenType::ZEROS) { - std::fill(int8_data.begin(), int8_data.end(), 0); - } else { - std::fill(int8_data.begin(), int8_data.end(), 0); - } - break; - } - case vkapi::kByte: { - uint8_data.resize(num_elements); - if (data_gen_type == DataGenType::RANDOM) { - generate_random_uint8_data(uint8_data, 0, 255, seed); - } else if (data_gen_type == DataGenType::RANDINT) { - generate_random_uint8_data(uint8_data, 0, 255, seed); - } else if (data_gen_type == DataGenType::RANDINT8) { - generate_random_uint8_data(uint8_data, 0, 255, seed); - } else if (data_gen_type == DataGenType::RANDINT4) { - generate_random_2xint4_data(uint8_data, seed); - } else if (data_gen_type == DataGenType::ONES) { - std::fill(uint8_data.begin(), uint8_data.end(), 1); - } else if (data_gen_type == DataGenType::ONES_INT4) { - uint8_t packed_data = (9 << 4) | 9; - std::fill(uint8_data.begin(), uint8_data.end(), packed_data); - } else if (data_gen_type == DataGenType::ZEROS) { - std::fill(uint8_data.begin(), uint8_data.end(), 0); - } else { - std::fill(uint8_data.begin(), uint8_data.end(), 0); - } - break; - } - default: - // Default to float - float_data.resize(num_elements); - if (data_gen_type == DataGenType::RANDOM) { - generate_random_float_data(float_data, -1.0f, 1.0f, seed); - } else if (data_gen_type == DataGenType::RANDINT) { - generate_randint_float_data(float_data, -10, 10, seed); - } else if (data_gen_type == DataGenType::ONES) { - generate_ones_data(float_data); - } else if (data_gen_type == DataGenType::ZEROS) { - generate_zeros_data(float_data); - } else { - generate_zeros_data(float_data); - } - break; - } -} - -int64_t ValueSpec::numel() const { - if (spec_type == SpecType::Int || spec_type == SpecType::Float || - spec_type == SpecType::Bool) { - return 1; - } else if (spec_type == SpecType::IntList) { - return sizes.empty() ? 0 : sizes[0]; - } else { // Tensor - int64_t total = 1; - for (int64_t size : sizes) { - total *= size; - } - return total; - } -} - -size_t ValueSpec::nbytes() const { - size_t element_size = 0; - switch (dtype) { - case vkapi::kFloat: - element_size = sizeof(float); - break; - case vkapi::kHalf: - element_size = sizeof(uint16_t); - break; - case vkapi::kInt: - element_size = sizeof(int32_t); - break; - case vkapi::kChar: - element_size = sizeof(int8_t); - break; - case vkapi::kByte: - element_size = sizeof(uint8_t); - break; - default: - element_size = sizeof(float); // Default fallback - break; - } - return numel() * element_size; -} - -std::string ValueSpec::to_string() const { - std::string result = "ValueSpec("; - - switch (spec_type) { - case SpecType::Tensor: - result += "type=Tensor, sizes=["; - break; - case SpecType::IntList: - result += "type=IntList, count="; - result += std::to_string(sizes.empty() ? 0 : sizes[0]); - result += ", data_gen="; - result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; - result += ")"; - return result; - case SpecType::Int: - result += "type=Int, value="; - result += std::to_string(get_int_value()); - result += ", data_gen="; - result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; - result += ")"; - return result; - case SpecType::Float: - result += "type=Float, value="; - result += std::to_string(get_float_value()); - result += ", data_gen="; - result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; - result += ")"; - return result; - case SpecType::Bool: - result += "type=Bool, value="; - result += get_bool_value() ? "true" : "false"; - result += ", data_gen="; - result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; - result += ")"; - return result; - case SpecType::String: - result += "type=String, value=\""; - result += get_string_value(); - result += "\")"; - return result; - } - - for (size_t i = 0; i < sizes.size(); ++i) { - result += std::to_string(sizes[i]); - if (i < sizes.size() - 1) - result += ", "; - } - result += "]"; - - if (spec_type == SpecType::Tensor) { - result += ", dtype="; - switch (dtype) { - case vkapi::kFloat: - result += "float"; - break; - case vkapi::kHalf: - result += "half"; - break; - case vkapi::kInt: - result += "int32"; - break; - case vkapi::kChar: - result += "int8"; - break; - case vkapi::kByte: - result += "uint8"; - break; - default: - result += "unknown"; - break; - } - - result += ", memory_layout="; - switch (memory_layout) { - case utils::kWidthPacked: - result += "WidthPacked"; - break; - case utils::kHeightPacked: - result += "HeightPacked"; - break; - case utils::kChannelsPacked: - result += "ChannelsPacked"; - break; - default: - result += "unknown"; - break; - } - - result += ", storage_type="; - switch (storage_type) { - case utils::kTexture3D: - result += "Texture3D"; - break; - case utils::kBuffer: - result += "Buffer"; - break; - default: - result += "unknown"; - break; - } - } - - result += ", data_gen="; - switch (data_gen_type) { - case DataGenType::FIXED: - result += "FIXED"; - break; - case DataGenType::RANDOM: - result += "RANDOM"; - break; - case DataGenType::RANDINT: - result += "RANDINT"; - break; - case DataGenType::RANDINT8: - result += "RANDINT8"; - break; - case DataGenType::RANDINT4: - result += "RANDINT4"; - break; - case DataGenType::ONES: - result += "ONES"; - break; - case DataGenType::ZEROS: - result += "ZEROS"; - break; - default: - result += "unknown"; - break; - } - result += ")"; - return result; -} - -// Additional ValueSpec methods -void ValueSpec::resize_data(size_t new_size) { - // Generate first so a deferred tensor keeps its data-gen pattern (resized, - // not pinned to zeros by the data_generated_ flag set below). - ensure_data_generated(); - ensure_unique_data(); - switch (dtype) { - case vkapi::kFloat: - data_->float_data.resize(new_size); - break; - case vkapi::kHalf: - data_->half_data.resize(new_size); - break; - case vkapi::kInt: - data_->int32_data.resize(new_size); - break; - case vkapi::kChar: - data_->int8_data.resize(new_size); - break; - case vkapi::kByte: - data_->uint8_data.resize(new_size); - break; - default: - data_->float_data.resize(new_size); - break; - } - data_generated_ = true; -} - -void* ValueSpec::get_mutable_data_ptr() { - ensure_data_generated(); - ensure_unique_data(); - switch (dtype) { - case vkapi::kFloat: - return data_->float_data.data(); - case vkapi::kHalf: - return data_->half_data.data(); - case vkapi::kInt: - return data_->int32_data.data(); - case vkapi::kChar: - return data_->int8_data.data(); - case vkapi::kByte: - return data_->uint8_data.data(); - default: - return data_->float_data.data(); - } -} - -float ValueSpec::get_element(size_t index) const { - ensure_data_generated(); - if (index >= static_cast(numel())) { - return 0.0f; - } - - switch (dtype) { - case vkapi::kFloat: - return index < data_->float_data.size() ? data_->float_data[index] : 0.0f; - case vkapi::kHalf: - return index < data_->half_data.size() - ? half_to_float(data_->half_data[index]) - : 0.0f; - case vkapi::kInt: - return index < data_->int32_data.size() - ? static_cast(data_->int32_data[index]) - : 0.0f; - case vkapi::kChar: - return index < data_->int8_data.size() - ? static_cast(data_->int8_data[index]) - : 0.0f; - case vkapi::kByte: - return index < data_->uint8_data.size() - ? static_cast(data_->uint8_data[index]) - : 0.0f; - default: - return 0.0f; - } -} - -const void* ValueSpec::get_data_ptr() const { - ensure_data_generated(); - switch (dtype) { - case vkapi::kFloat: - return data_->float_data.data(); - case vkapi::kHalf: - return data_->half_data.data(); - case vkapi::kInt: - return data_->int32_data.data(); - case vkapi::kChar: - return data_->int8_data.data(); - case vkapi::kByte: - return data_->uint8_data.data(); - default: - throw std::runtime_error("Unsupported data type for get_data_ptr"); - } -} - -void generate_random_float_data( - std::vector& data, - float min_val, - float max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_real_distribution dis(min_val, max_val); - for (auto& val : data) { - val = dis(gen); - } -} - -void generate_random_int_data( - std::vector& data, - int min_val, - int max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(min_val, max_val); - for (auto& val : data) { - val = dis(gen); - } -} - -void generate_randint_float_data( - std::vector& data, - int min_val, - int max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(min_val, max_val); - for (auto& val : data) { - val = static_cast(dis(gen)); - } -} - -void generate_randint_half_data( - std::vector& data, - int min_val, - int max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(min_val, max_val); - for (auto& val : data) { - val = float_to_half(static_cast(dis(gen))); - } -} - -void generate_ones_data(std::vector& data) { - std::fill(data.begin(), data.end(), 1.0f); -} - -void generate_random_int8_data( - std::vector& data, - int8_t min_val, - int8_t max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(min_val, max_val); - for (auto& val : data) { - val = static_cast(dis(gen)); - } -} - -void generate_random_uint8_data( - std::vector& data, - uint8_t min_val, - uint8_t max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(min_val, max_val); - for (auto& val : data) { - val = static_cast(dis(gen)); - } -} - -void generate_random_int4_data( - std::vector& data, - int8_t min_val, - int8_t max_val, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(min_val, max_val); - for (auto& val : data) { - val = static_cast(dis(gen)); - } -} - -void generate_random_2xint4_data(std::vector& data, int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(-8, 7); // Signed 4-bit range - for (auto& val : data) { - // Generate two separate 4-bit values - int8_t lower_4bits = static_cast(dis(gen)) & 0x0F; - int8_t upper_4bits = static_cast(dis(gen)) & 0x0F; - // Pack them into a single 8-bit value - val = (upper_4bits << 4) | lower_4bits; - } -} - -void generate_random_2xint4_data( - std::vector& data, - int explicit_seed) { - std::mt19937 gen(get_seed_or_explicit(explicit_seed)); - std::uniform_int_distribution dis(0, 15); // Unsigned 4-bit range - for (auto& val : data) { - // Generate two separate 4-bit values - uint8_t lower_4bits = static_cast(dis(gen)) & 0x0F; - uint8_t upper_4bits = static_cast(dis(gen)) & 0x0F; - // Pack them into a single 8-bit value - val = (upper_4bits << 4) | lower_4bits; - } -} - -void generate_zeros_data(std::vector& data) { - std::fill(data.begin(), data.end(), 0.0f); -} - -// Correctness checking against reference data -bool ValueSpec::validate_against_reference( - float abs_tolerance, - float rel_tolerance) const { - // Only validate float and half tensors. For half tensors, convert the - // computed half data to float for comparison against the fp32 reference. - if (!is_tensor() || (dtype != vkapi::kFloat && dtype != vkapi::kHalf)) { - return true; // Skip validation for non-float/half or non-tensor types - } - - // For kHalf, materialize the GPU output as float so the same tolerance - // machinery can compare against the (always-float) reference data. - std::vector half_as_float; - if (dtype == vkapi::kHalf) { - const auto& half_bits = get_half_data(); - half_as_float.resize(half_bits.size()); - for (size_t i = 0; i < half_bits.size(); ++i) { - half_as_float[i] = half_to_float(half_bits[i]); - } - } - // Materialize computed data as float32 for comparison. The dtype is - // guaranteed to be float or half by the early-out above. - const std::vector& computed_data = - (dtype == vkapi::kHalf) ? half_as_float : get_float_data(); - const auto& reference_data = get_ref_float_data(); - - // Skip validation if no reference data is available - if (reference_data.empty()) { - return true; - } - - // Check if sizes match - if (computed_data.size() != reference_data.size()) { - if (debugging()) { - std::cout << "Size mismatch: computed=" << computed_data.size() - << ", reference=" << reference_data.size() << std::endl; - } - return false; - } - - // Element-wise comparison with both absolute and relative tolerance - size_t num_mismatched = 0; - size_t first_mismatch = 0; - for (size_t i = 0; i < computed_data.size(); ++i) { - float diff = std::abs(computed_data[i] - reference_data[i]); - float abs_ref = std::abs(reference_data[i]); - - // Check if either absolute or relative tolerance condition is satisfied - bool abs_tolerance_ok = diff <= abs_tolerance; - bool rel_tolerance_ok = diff <= rel_tolerance * abs_ref; - - if (!abs_tolerance_ok && !rel_tolerance_ok) { - if (num_mismatched == 0) { - first_mismatch = i; - std::cout << "Mismatch at element " << i - << ": computed=" << computed_data[i] - << ", reference=" << reference_data[i] << ", diff=" << diff - << ", abs_tolerance=" << abs_tolerance - << ", rel_tolerance=" << rel_tolerance - << ", rel_threshold=" << (rel_tolerance * abs_ref) - << std::endl; - } - num_mismatched++; - } - } - if (num_mismatched > 0) { - std::cout << " total mismatched: " << num_mismatched << " / " - << computed_data.size() << " (first at " << first_mismatch << ")" - << std::endl; - // For 2D outputs, print a per-16x16-tile mismatch-count map to expose - // the spatial structure of the failure (e.g. zeroed MMA subtiles). - if (sizes.size() == 2) { - const int64_t Mr = sizes[0]; - const int64_t Nc = sizes[1]; - std::cout << " 16x16-tile mismatch counts (rows=M/16, cols=N/16):" - << std::endl; - for (int64_t ti = 0; ti < (Mr + 15) / 16; ++ti) { - std::cout << " "; - for (int64_t tj = 0; tj < (Nc + 15) / 16; ++tj) { - int count = 0; - for (int64_t r = ti * 16; r < std::min(Mr, (ti + 1) * 16); ++r) { - for (int64_t c = tj * 16; c < std::min(Nc, (tj + 1) * 16); ++c) { - float diff = std::abs( - computed_data[r * Nc + c] - reference_data[r * Nc + c]); - float abs_ref = std::abs(reference_data[r * Nc + c]); - if (diff > abs_tolerance && diff > rel_tolerance * abs_ref) { - count++; - } - } - } - std::cout << std::setw(4) << count; - } - std::cout << std::endl; - } - } - return false; - } - - if (debugging()) { - std::cout << "Correctness validation PASSED" << std::endl; - } - return true; -} - -// Ensure data is generated for this ValueSpec -void ValueSpec::ensure_data_generated(int seed) const { - if (data_generated_) { - return; - } - generate_tensor_data(seed); - data_generated_ = true; -} - -void ValueSpec::share_data_from(const ValueSpec& other) { - if (!is_tensor() || !other.is_tensor()) { - return; - } - // Materialize the source first: sharing an ungenerated payload would let a - // later access materialize each spec independently under different seeds. - other.ensure_data_generated(); - data_ = other.data_; - data_generated_ = true; -} - -void ValueSpec::share_reference_from(const ValueSpec& other) { - if (!is_tensor() || !other.is_tensor()) { - return; - } - reference_data_ = other.reference_data_; -} - -// ReferenceKey implementation -ReferenceKey ReferenceKey::from_test_case(const TestCase& tc) { - std::ostringstream oss; - - // Serialize inputs that affect reference computation - // Skip: storage_type, memory_layout, string values (like impl_selector) - for (size_t i = 0; i < tc.inputs().size(); ++i) { - const ValueSpec& input = tc.inputs()[i]; - oss << "i" << i << ":"; - - if (input.is_tensor()) { - // For tensors: sizes, dtype, data_gen_type, is_constant - oss << "T["; - for (size_t j = 0; j < input.sizes.size(); ++j) { - if (j > 0) - oss << ","; - oss << input.sizes[j]; - } - oss << "]d" << static_cast(input.dtype); - oss << "g" << static_cast(input.data_gen_type); - oss << "c" << (input.is_constant() ? 1 : 0); - oss << "n" << (input.is_none() ? 1 : 0); - } else if (input.is_int()) { - oss << "I" << input.get_int_value(); - } else if (input.is_float()) { - oss << "F" << input.get_float_value(); - } else if (input.is_bool()) { - oss << "B" << (input.get_bool_value() ? 1 : 0); - } else if (input.is_int_list()) { - oss << "L["; - const auto& list = input.get_int_list(); - for (size_t j = 0; j < list.size(); ++j) { - if (j > 0) - oss << ","; - oss << list[j]; - } - oss << "]"; - } - // Skip string inputs (like impl_selector) as they don't affect reference - oss << ";"; - } - - // Also include output shapes for completeness - for (size_t i = 0; i < tc.outputs().size(); ++i) { - const ValueSpec& output = tc.outputs()[i]; - oss << "o" << i << ":["; - for (size_t j = 0; j < output.sizes.size(); ++j) { - if (j > 0) - oss << ","; - oss << output.sizes[j]; - } - oss << "]d" << static_cast(output.dtype) << ";"; - } - - ReferenceKey key; - key.key_string = oss.str(); - return key; -} - -// Helper function to collect GPU timing from querypool -float collect_gpu_timing_us( - ComputeGraph& graph, - const std::vector& shader_filter) { - graph.context()->querypool().extract_results(); - const auto results = graph.context()->querypool().get_shader_timestamp_data(); - if (!results.empty()) { - // Sum durations of all shaders that don't match any pattern in - // shader_filter - float total_duration_us = 0.0f; - for (const auto& shader_result : results) { - bool filtered = false; - // Check if this shader matches any filter pattern - for (const auto& filter_pattern : shader_filter) { - if (shader_result.kernel_name.find(filter_pattern) != - std::string::npos) { - filtered = true; - break; - } - } - - if (!filtered) { - // Calculate duration from start and end times, convert from ns to μs - uint64_t duration_ns = - shader_result.end_time_ns - shader_result.start_time_ns; - total_duration_us += static_cast(duration_ns) / 1000.0f; - } - } - return total_duration_us; - } - return 0.0f; -} - -// Helper function to collect per-shader GPU timing from querypool -// Returns a map of shader_name -> timing_us for non-filtered shaders -std::unordered_map collect_per_shader_timing_us( - ComputeGraph& graph, - const std::vector& shader_filter) { - std::unordered_map shader_timings; - - graph.context()->querypool().extract_results(); - const auto results = graph.context()->querypool().get_shader_timestamp_data(); - for (const auto& shader_result : results) { - bool filtered = false; - // Check if this shader matches any filter pattern - for (const auto& filter_pattern : shader_filter) { - if (shader_result.kernel_name.find(filter_pattern) != std::string::npos) { - filtered = true; - break; - } - } - - if (!filtered) { - // Calculate duration from start and end times, convert from ns to μs - uint64_t duration_ns = - shader_result.end_time_ns - shader_result.start_time_ns; - float duration_us = static_cast(duration_ns) / 1000.0f; - // Accumulate timing for shaders with the same name - shader_timings[shader_result.kernel_name] += duration_us; - } - } - return shader_timings; -} - -// BenchmarkResult implementation -void BenchmarkResult::add_iter_timing(float time_us) { - iter_timings.push_back(time_us); -} - -void BenchmarkResult::add_shader_timing( - const std::string& shader_name, - float time_us, - const uint32_t gwg[3], - const uint32_t lwg[3]) { - // Find existing shader timing or create new one - for (auto& st : shader_timings_) { - if (st.shader_name == shader_name) { - st.iter_timings_us.push_back(time_us); - // Work group sizes should be consistent across iterations - return; - } - } - // Not found, create new entry - ShaderTiming new_timing; - new_timing.shader_name = shader_name; - new_timing.iter_timings_us.push_back(time_us); - new_timing.gwg[0] = gwg[0]; - new_timing.gwg[1] = gwg[1]; - new_timing.gwg[2] = gwg[2]; - new_timing.lwg[0] = lwg[0]; - new_timing.lwg[1] = lwg[1]; - new_timing.lwg[2] = lwg[2]; - shader_timings_.push_back(std::move(new_timing)); -} - -float BenchmarkResult::get_avg_time_us() const { - if (iter_timings.empty()) { - return 0.0f; - } - - float sum = 0.0f; - for (float timing : iter_timings) { - sum += timing; - } - return sum / iter_timings.size(); -} - -float BenchmarkResult::get_min_time_us() const { - if (iter_timings.empty()) { - return 0.0f; - } - - return *std::min_element(iter_timings.begin(), iter_timings.end()); -} - -float BenchmarkResult::get_max_time_us() const { - if (iter_timings.empty()) { - return 0.0f; - } - - return *std::max_element(iter_timings.begin(), iter_timings.end()); -} - -float BenchmarkResult::get_std_dev_us() const { - if (iter_timings.size() <= 1) { - return 0.0f; - } - - float mean = get_avg_time_us(); - float sum_sq_diff = 0.0f; - - for (float timing : iter_timings) { - float diff = timing - mean; - sum_sq_diff += diff * diff; - } - - return std::sqrt(sum_sq_diff / (iter_timings.size() - 1)); -} - -void BenchmarkResult::print_summary( - int case_number, - const std::string& size_info, - float total_gflops) const { - static constexpr int OPERATOR_NAME_WIDTH = 50; - static constexpr int GLOBAL_WG_WIDTH = 16; - static constexpr int LOCAL_WG_WIDTH = 12; - static constexpr int KERNEL_NAME_WIDTH = 80; - static constexpr int SIZE_INFO_WIDTH = 20; - static constexpr int TIMING_WIDTH = 16; - static constexpr int GFLOPS_WIDTH = 14; - static constexpr int CORRECTNESS_WIDTH = 8; - - // Helper to truncate shader names longer than 46 chars to 44 chars + ".." - auto truncate_shader_name = [](const std::string& name) -> std::string { - if (name.length() > 46) { - return name.substr(0, 44) + ".."; - } - return name; - }; - - // Helper to format work group size as (x,y,z) - auto format_wg_size = [](const uint32_t wg[3]) -> std::string { - return "(" + std::to_string(wg[0]) + "," + std::to_string(wg[1]) + "," + - std::to_string(wg[2]) + ")"; - }; - - std::string correctness_str; - switch (correctness_status_) { - case CorrectnessStatus::SKIPPED: - correctness_str = "SKIPPED"; - break; - case CorrectnessStatus::PASSED: - correctness_str = "PASSED"; - break; - case CorrectnessStatus::FAILED: - correctness_str = "FAILED"; - break; - } - - // If we have per-shader timing data, print one line per shader plus overall - if (!shader_timings_.empty()) { - // If only one shader, print a single combined row - if (shader_timings_.size() == 1) { - const auto& st = shader_timings_[0]; - std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) - << truncate_shader_name(st.shader_name) << " " << std::left - << std::setw(GLOBAL_WG_WIDTH) << format_wg_size(st.gwg) - << std::left << std::setw(LOCAL_WG_WIDTH) - << format_wg_size(st.lwg) << std::left - << std::setw(KERNEL_NAME_WIDTH) << get_kernel_name() - << std::right << " " << std::setw(SIZE_INFO_WIDTH) << size_info - << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) - << get_avg_time_us() << " μs " << std::setw(GFLOPS_WIDTH) - << std::fixed << std::setprecision(3) << total_gflops - << " GFLOP/s " << std::setw(CORRECTNESS_WIDTH) - << correctness_str << std::endl; - } else { - // Multiple shaders: print individual shader lines (without GFLOP/s) - for (size_t i = 0; i < shader_timings_.size(); ++i) { - const auto& st = shader_timings_[i]; - float shader_avg_time = st.get_avg_time_us(); - - // Shader lines don't show test case info - std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) - << truncate_shader_name(st.shader_name) << " " << std::left - << std::setw(GLOBAL_WG_WIDTH) << format_wg_size(st.gwg) - << std::left << std::setw(LOCAL_WG_WIDTH) - << format_wg_size(st.lwg) << std::left - << std::setw(KERNEL_NAME_WIDTH) << "" << std::right << " " - << std::setw(SIZE_INFO_WIDTH) << "" << std::setw(TIMING_WIDTH) - << std::fixed << std::setprecision(3) << shader_avg_time - << " μs " << std::setw(GFLOPS_WIDTH) << "" << " " - << std::setw(CORRECTNESS_WIDTH) << "" << std::endl; - } - - // Print overall row with operator name, test case info, total time, and - // GFLOP/s - std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) - << get_operator_name() << " " << std::left - << std::setw(GLOBAL_WG_WIDTH) << "" << std::left - << std::setw(LOCAL_WG_WIDTH) << "" << std::left - << std::setw(KERNEL_NAME_WIDTH) << get_kernel_name() - << std::right << " " << std::setw(SIZE_INFO_WIDTH) << size_info - << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) - << get_avg_time_us() << " μs " << std::setw(GFLOPS_WIDTH) - << std::fixed << std::setprecision(3) << total_gflops - << " GFLOP/s " << std::setw(CORRECTNESS_WIDTH) - << correctness_str << std::endl; - } - - // Print separator line between test cases - } else { - // No per-shader timing data, use the original format - std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) - << get_operator_name() << " " << std::left - << std::setw(GLOBAL_WG_WIDTH) << "" << std::left - << std::setw(LOCAL_WG_WIDTH) << "" << std::left - << std::setw(KERNEL_NAME_WIDTH) << get_kernel_name() << std::right - << " " << std::setw(SIZE_INFO_WIDTH) << size_info - << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) - << get_avg_time_us() << " μs " << std::setw(GFLOPS_WIDTH) - << std::fixed << std::setprecision(3) << total_gflops - << " GFLOP/s " << std::setw(CORRECTNESS_WIDTH) << correctness_str - << std::endl; - } -} - -// TestResult implementation -void TestResult::add_result(const BenchmarkResult& result) { - results_.push_back(result); -} - -void TestResult::add_result(BenchmarkResult&& result) { - results_.push_back(std::move(result)); -} - -void TestResult::print_summary() const { - static constexpr int CASE_WIDTH = 100; - static constexpr int KERNEL_NAME_WIDTH = 20; - static constexpr int TIMING_WIDTH = 12; - static constexpr int PASS_WIDTH = 8; - - if (results_.empty()) { - std::cout << "No results to display" << std::endl; - return; - } - - std::cout << "\n=== " << operation_name_ - << " Performance Summary ===" << std::endl; - print_separator(); - - std::cout << std::left << std::setw(CASE_WIDTH) << "Case" << std::left - << std::setw(KERNEL_NAME_WIDTH) << "Kernel Name" << std::left - << std::setw(TIMING_WIDTH) << "Avg (μs)" << std::left - << std::setw(TIMING_WIDTH) << "Min (μs)" << std::left - << std::setw(TIMING_WIDTH) << "Max (μs)" << std::left - << std::setw(TIMING_WIDTH) << "Std Dev" << std::left - << std::setw(PASS_WIDTH) << "Pass" << std::endl; - print_separator(); - - for (size_t i = 0; i < results_.size(); ++i) { - const auto& result = results_[i]; - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - std::cout << std::left << std::setw(CASE_WIDTH) << i + 1 << std::left - << std::setw(KERNEL_NAME_WIDTH) - << result.get_kernel_name().substr(0, KERNEL_NAME_WIDTH - 1) - << std::left << std::setw(TIMING_WIDTH) << std::fixed - << std::setprecision(3) << result.get_avg_time_us() << std::left - << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) - << result.get_min_time_us() << std::left - << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) - << result.get_max_time_us() << std::left - << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) - << result.get_std_dev_us() << std::left << std::setw(PASS_WIDTH) - << (vulkan_execute_succeeded ? "✓" : "✗") << std::endl; - } - - print_separator(); - std::cout << "Total cases: " << results_.size() - << ", Passed: " << get_passed_count() - << ", Failed: " << get_failed_count() << std::endl; - std::cout << "Overall GFLOP/s: " << std::fixed << std::setprecision(3) - << gflops_ << std::endl; - std::cout << "Overall correctness: " - << (correctness_passed_ ? "PASSED" : "FAILED") << std::endl; -} - -void TestResult::print_detailed_results() const { - if (results_.empty()) { - std::cout << "No results to display" << std::endl; - return; - } - - std::cout << "\n=== " << operation_name_ - << " Detailed Results ===" << std::endl; - - for (size_t i = 0; i < results_.size(); ++i) { - const auto& result = results_[i]; - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - std::cout << "\nCase " << i + 1 << ": " << result.get_kernel_name() - << std::endl; - std::cout << " Iterations: " << result.get_num_iterations() << std::endl; - std::cout << " Average: " << std::fixed << std::setprecision(3) - << result.get_avg_time_us() << " μs" << std::endl; - std::cout << " Min: " << std::fixed << std::setprecision(3) - << result.get_min_time_us() << " μs" << std::endl; - std::cout << " Max: " << std::fixed << std::setprecision(3) - << result.get_max_time_us() << " μs" << std::endl; - std::cout << " Std Dev: " << std::fixed << std::setprecision(3) - << result.get_std_dev_us() << " μs" << std::endl; - std::cout << " Correctness: " - << (vulkan_execute_succeeded ? "PASSED" : "FAILED") << std::endl; - - if (result.get_num_iterations() > 0) { - std::cout << " Individual timings (μs): "; - const auto& timings = result.get_iter_timings(); - for (size_t j = 0; j < std::min(size_t(10), timings.size()); ++j) { - std::cout << std::fixed << std::setprecision(1) << timings[j]; - if (j < std::min(size_t(10), timings.size()) - 1) - std::cout << ", "; - } - if (timings.size() > 10) { - std::cout << " ... (" << (timings.size() - 10) << " more)"; - } - std::cout << std::endl; - } - } - - std::cout << "\nOverall Results:" << std::endl; - std::cout << " Total GFLOP/s: " << std::fixed << std::setprecision(3) - << gflops_ << std::endl; - std::cout << " Overall correctness: " - << (correctness_passed_ ? "PASSED" : "FAILED") << std::endl; -} - -void TestResult::print_statistics() const { - if (results_.empty()) { - std::cout << "No results to display statistics for" << std::endl; - return; - } - - std::cout << "\n=== " << operation_name_ << " Statistics ===" << std::endl; - std::cout << "Total test cases: " << results_.size() << std::endl; - std::cout << "Passed: " << get_passed_count() << std::endl; - std::cout << "Failed: " << get_failed_count() << std::endl; - std::cout << "Success rate: " << std::fixed << std::setprecision(1) - << (100.0f * get_passed_count() / results_.size()) << "%" - << std::endl; - - if (get_passed_count() > 0) { - std::cout << "Total average time: " << std::fixed << std::setprecision(3) - << get_total_avg_time_us() << " μs" << std::endl; - std::cout << "Total GFLOP/s: " << std::fixed << std::setprecision(3) - << get_total_gflops() << std::endl; - - const auto* fastest = get_fastest_result(); - const auto* slowest = get_slowest_result(); - const auto* highest_gflops = get_highest_gflops_result(); - - if (fastest) { - std::cout << "Fastest case: " << fastest->get_kernel_name() << " (" - << std::fixed << std::setprecision(3) - << fastest->get_avg_time_us() << " μs)" << std::endl; - } - - if (slowest) { - std::cout << "Slowest case: " << slowest->get_kernel_name() << " (" - << std::fixed << std::setprecision(3) - << slowest->get_avg_time_us() << " μs)" << std::endl; - } - - if (highest_gflops) { - std::cout << "Best performing case: " << highest_gflops->get_kernel_name() - << " (" << std::fixed << std::setprecision(3) - << highest_gflops->get_avg_time_us() << " μs)" << std::endl; - } - } -} - -void TestResult::print_brief_summary() const { - print_separator(); - std::cout << "Summary Statistics:" << std::endl; - - if (get_passed_count() > 0) { - std::cout << "Average execution time: " << std::fixed - << std::setprecision(3) << get_total_avg_time_us() << " μs" - << std::endl; - std::cout << "Total throughput: " << std::fixed << std::setprecision(3) - << get_gflops() << " GFLOP/s" << std::endl; - std::cout << "Successful test cases: " << get_passed_count() << "/" - << size() << std::endl; - std::cout << "Overall correctness: " - << (get_correctness_passed() ? "PASSED" : "FAILED") << std::endl; - } else { - std::cout << "No successful test cases to report" << std::endl; - } -} - -float TestResult::get_total_avg_time_us() const { - if (results_.empty()) { - return 0.0f; - } - - float sum = 0.0f; - size_t count = 0; - - for (const auto& result : results_) { - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - if (vulkan_execute_succeeded) { - sum += result.get_avg_time_us(); - count++; - } - } - - return count > 0 ? sum / count : 0.0f; -} - -float TestResult::get_total_gflops() const { - return gflops_; -} - -size_t TestResult::get_passed_count() const { - size_t count = 0; - for (const auto& result : results_) { - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - if (vulkan_execute_succeeded) { - count++; - } - } - return count; -} - -size_t TestResult::get_failed_count() const { - return results_.size() - get_passed_count(); -} - -const BenchmarkResult* TestResult::get_fastest_result() const { - const BenchmarkResult* fastest = nullptr; - - for (const auto& result : results_) { - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - if (vulkan_execute_succeeded) { - if (!fastest || result.get_avg_time_us() < fastest->get_avg_time_us()) { - fastest = &result; - } - } - } - - return fastest; -} - -const BenchmarkResult* TestResult::get_slowest_result() const { - const BenchmarkResult* slowest = nullptr; - - for (const auto& result : results_) { - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - if (vulkan_execute_succeeded) { - if (!slowest || result.get_avg_time_us() > slowest->get_avg_time_us()) { - slowest = &result; - } - } - } - - return slowest; -} - -const BenchmarkResult* TestResult::get_highest_gflops_result() const { - // Since GFLOPS is now a TestResult-level metric rather than per-case, - // this method returns the fastest result as a proxy for highest performance - return get_fastest_result(); -} - -// Default FLOP calculation function (assumes 1 FLOP per element) -int64_t default_flop_calculator(const TestCase& test_case) { - // Calculate total elements from the first input tensor - int64_t total_elements = 1; - if (!test_case.empty() && test_case.num_inputs() > 0 && - test_case.inputs()[0].is_tensor()) { - const auto& sizes = test_case.inputs()[0].get_tensor_sizes(); - for (int64_t size : sizes) { - total_elements *= size; - } - } - - // Assume 1 FLOP per element for basic operations - return total_elements; -} - -BenchmarkGraph setup_compute_graph( - TestCase& test_case, - std::string op_name, - int op_invocations_per_execute) { - GraphConfig config; - config.enable_querypool = true; - // Pool sizing takes max(execute, prepack) * factor, so scaling the factor - // also over-reserves the prepack side (encoded once). Accepted: precise - // execute-only sizing would need runtime changes. - config.descriptor_pool_safety_factor *= - std::max(1, op_invocations_per_execute); - // Default-on (opt-out via TestCase::set_force_resize(false)): force every - // DynamicDispatchNode to run its resize function when execute_test_case - // runs propagate_resize() after prepack, exercising the op's resize formula - // even when input shapes are unchanged. - config.force_resize = test_case.get_force_resize(); - config.force_narrow_int4_tile = test_case.get_force_narrow_int4_tile(); - auto graph = std::make_unique(config); - - std::vector input_values; - - // Process input ValueSpecs - for (size_t i = 0; i < test_case.num_inputs(); ++i) { - const ValueSpec& input_spec = test_case.inputs()[i]; - - if (input_spec.is_none()) { - input_values.push_back(graph->add_none()); - } else if (input_spec.is_float()) { - ValueRef input_value = - graph->add_scalar(static_cast(input_spec.get_float_value())); - input_values.push_back(input_value); - } else if (input_spec.is_int()) { - ValueRef input_value = - graph->add_scalar(static_cast(input_spec.get_int_value())); - input_values.push_back(input_value); - } else if (input_spec.is_bool()) { - ValueRef input_value = graph->add_scalar(input_spec.get_bool_value()); - input_values.push_back(input_value); - } else if (input_spec.is_int_list()) { - // Convert int32_t list to int64_t list for ComputeGraph - const auto& int32_list = input_spec.get_int_list(); - std::vector int64_list; - int64_list.reserve(int32_list.size()); - for (int32_t val : int32_list) { - int64_list.push_back(static_cast(val)); - } - ValueRef input_value = graph->add_scalar_list(std::move(int64_list)); - input_values.push_back(input_value); - } else if (input_spec.is_string()) { - std::string str_copy = input_spec.get_string_value(); - ValueRef input_value = graph->add_string(std::move(str_copy)); - input_values.push_back(input_value); - } else if (input_spec.is_constant()) { - ValueRef input_value = graph->add_tensorref( - input_spec.get_tensor_sizes(), - input_spec.dtype, - input_spec.get_data_ptr()); - input_values.push_back(input_value); - } else { - IOValueRef input_io = graph->add_input_tensor( - input_spec.get_tensor_sizes(), - input_spec.dtype, - input_spec.storage_type, - input_spec.memory_layout); - input_values.push_back(input_io.value); - } - } - - std::vector output_values; - - // Process output ValueSpecs - for (size_t i = 0; i < test_case.num_outputs(); ++i) { - const ValueSpec& output_spec = test_case.outputs()[i]; - - if (!output_spec.is_tensor()) { - throw std::runtime_error("All output specifications must be tensors"); - } - - // Create output tensor - ValueRef output_value = graph->add_tensor( - output_spec.get_tensor_sizes(), - output_spec.dtype, - output_spec.storage_type, - output_spec.memory_layout); - - output_values.push_back(output_value); - } - - // Get the operator function and call it - auto opFn = VK_GET_OP_FN(op_name); - - // Create arguments vector for the operator function - std::vector op_args = input_values; - op_args.insert(op_args.end(), output_values.begin(), output_values.end()); - - // Nodes added before the op are staging uploads; nodes added after are - // staging downloads. Only the op's own nodes are repeated by benchmarks. - const size_t op_begin = graph->execute_nodes().size(); - opFn(*graph, op_args); - const size_t op_end = graph->execute_nodes().size(); - - for (size_t i = 0; i < output_values.size(); ++i) { - graph->set_output_value(output_values[i]); - } - return {std::move(graph), {op_begin, op_end}}; -} - -// Test execution utilities -BenchmarkResult execute_test_case( - TestCase& test_case, - int warmup_runs, - int benchmark_runs, - int chained_dispatches, - bool write_outputs) { - BenchmarkResult result( - test_case.name().empty() ? "unnamed_test_case" : test_case.name()); - - // Initialize querypool if using GPU timestamps - if (use_gpu_timestamps()) { - api::context()->initialize_querypool(); - } - - // Build the operator once. Benchmark repetition is encoded separately so - // persistent graph allocations are not duplicated. - BenchmarkGraph benchmark = setup_compute_graph( - test_case, test_case.operator_name(), chained_dispatches); - ComputeGraph& graph = *benchmark.graph; - - // Prepare the graph - graph.prepare(); - graph.prepack(); - - // Run resize functions once so force_resize exercises resize formulas even - // though the record/replay path below never calls propagate_resize(). - graph.propagate_resize(); - - // Copy input data into the graph's staging buffers - size_t graph_input_idx = 0; - for (size_t i = 0; i < test_case.num_inputs(); ++i) { - const ValueSpec& input_spec = test_case.inputs()[i]; - - // Only non-constant tensor inputs correspond to graph.inputs() entries - bool is_graph_input = input_spec.is_tensor() && !input_spec.is_constant() && - !input_spec.is_none(); - if (!is_graph_input) { - continue; - } - - if (graph_input_idx < graph.inputs().size()) { - const auto& input_ref = graph.inputs()[graph_input_idx]; - - // Get the appropriate data based on dtype - const void* data_ptr = nullptr; - size_t data_numel = input_spec.numel(); - - switch (input_spec.dtype) { - case vkapi::kFloat: - data_ptr = input_spec.get_float_data().data(); - break; - case vkapi::kHalf: - data_ptr = input_spec.get_half_data().data(); - break; - case vkapi::kInt: - data_ptr = input_spec.get_int32_data().data(); - break; - case vkapi::kChar: - data_ptr = input_spec.get_int8_data().data(); - break; - case vkapi::kByte: - data_ptr = input_spec.get_uint8_data().data(); - break; - default: - throw std::runtime_error("Unsupported data type for input tensor"); - } - - // Copy data into staging buffer - graph.maybe_cast_and_copy_into_staging( - input_ref.staging, data_ptr, data_numel, input_spec.dtype); - } - ++graph_input_idx; - } - - RepeatedGraphExecutor graph_executor( - graph, chained_dispatches, benchmark.op_nodes); - - // Warmup runs - for (int run = 0; run < warmup_runs; ++run) { - graph_executor.execute(); - } - - // Benchmark runs - collect individual iteration timings - float total_cpu_time_us = 0.0f; - float total_gpu_time_us = 0.0f; - - const float chained_dispatches_f = static_cast(chained_dispatches); - - for (int run = 0; run < benchmark_runs; ++run) { - // Measure CPU time for each execute() call - auto cpu_start = std::chrono::high_resolution_clock::now(); - graph_executor.execute(); - auto cpu_end = std::chrono::high_resolution_clock::now(); - - auto cpu_duration = std::chrono::duration_cast( - cpu_end - cpu_start); - // Each execute() ran chained_dispatches op invocations; report - // per-invocation time. - float cpu_time_us = - static_cast(cpu_duration.count()) / chained_dispatches_f; - total_cpu_time_us += cpu_time_us; - - // Collect per-shader GPU timing - get raw shader results to preserve - // metadata - graph.context()->querypool().extract_results(); - const auto shader_results = - graph.context()->querypool().get_shader_timestamp_data(); - - // Calculate total GPU time from per-shader timings - float gpu_time_us = 0.0f; - for (const auto& shader_result : shader_results) { - // Check if this shader matches any filter pattern - bool filtered = false; - for (const auto& filter_pattern : test_case.get_shader_filter()) { - if (shader_result.kernel_name.find(filter_pattern) != - std::string::npos) { - filtered = true; - break; - } - } - - if (!filtered) { - uint64_t duration_ns = - shader_result.end_time_ns - shader_result.start_time_ns; - float duration_us = static_cast(duration_ns) / 1000.0f; - gpu_time_us += duration_us; - // Per-shader sample is already one querypool entry per dispatch (at - // chained_dispatches>1 we get that many entries per shader), so do NOT - // divide here — the aggregator (get_avg_time_us) averages across them - // naturally. - result.add_shader_timing( - shader_result.kernel_name, - duration_us, - shader_result.metadata.gwg, - shader_result.metadata.lwg); - } - } - // gpu_time_us aggregates chained_dispatches worth of shader runs; divide - // it down to per-invocation time. - gpu_time_us /= chained_dispatches_f; - total_gpu_time_us += gpu_time_us; - - // Add the appropriate timing based on the flag - float iter_time_us = use_gpu_timestamps() ? gpu_time_us : cpu_time_us; - result.add_iter_timing(iter_time_us); - } - - // Calculate averages for display - float avg_cpu_time_us = total_cpu_time_us / benchmark_runs; - float avg_gpu_time_us = total_gpu_time_us / benchmark_runs; - - // Print both timings if latency printing is enabled - if (print_latencies()) { - if (use_gpu_timestamps()) { - graph.context()->querypool().print_results(); - } - std::cout << " CPU timing: " << std::fixed << std::setprecision(3) - << avg_cpu_time_us << " μs" << std::endl; - std::cout << " GPU timing: " << std::fixed << std::setprecision(3) - << avg_gpu_time_us << " μs" << std::endl; - std::cout << " Using " << (use_gpu_timestamps() ? "GPU" : "CPU") - << " timing for result" << std::endl; - } - - // Copy output data from the graph's staging buffers. The benchmarking path - // skips this work since it doesn't need correctness data — the probe pass - // has already populated test_case.outputs() at chained_dispatches=1. - if (write_outputs) { - for (size_t i = 0; i < test_case.num_outputs(); ++i) { - ValueSpec& output_spec = test_case.outputs()[i]; - - if (output_spec.is_tensor() && i < graph.outputs().size()) { - const auto& output_ref = graph.outputs()[i]; - - // Ensure output data vector is properly sized - size_t data_numel = output_spec.numel(); - output_spec.resize_data(data_numel); - - // Get mutable data pointer for the output - void* data_ptr = output_spec.get_mutable_data_ptr(); - - if (data_ptr != nullptr) { - // Copy data from staging buffer to output spec - graph.maybe_cast_and_copy_from_staging( - output_ref.staging, data_ptr, data_numel, output_spec.dtype); - } - - // Print output tensor data if output printing is enabled - if (print_output()) { - std::string output_name = "Output[" + std::to_string(i) + "]"; - print_valuespec_data(output_spec, output_name); - } - } - } - } - - return result; -} - -TestResult execute_test_cases( - std::function()> test_case_generator, - FlopCalculatorFunc flop_calculator, - const std::string& operation_name, - int warmup_runs, - int benchmark_runs, - ReferenceComputeFunc reference_compute_func) { - TestResult results(operation_name); - - // Generate all test cases - std::vector test_cases = test_case_generator(); - - std::cout << "Executing " << test_cases.size() << " test cases for " - << operation_name << std::endl; - print_separator(); - - // Group test cases by ReferenceKey for caching reference computations - // Use a vector to preserve the order in which groups first appear - std::vector group_order; - std::unordered_map, ReferenceKeyHash> - groups; - for (size_t i = 0; i < test_cases.size(); ++i) { - ReferenceKey key = ReferenceKey::from_test_case(test_cases[i]); - if (groups.find(key) == groups.end()) { - group_order.push_back(key); - } - groups[key].push_back(i); - } - - bool any_correctness_failed = false; - float total_gflops = 0.0f; - size_t test_case_counter = 0; - - // Process each group: generate data, compute reference, execute, and print - // Iterate in the order groups first appeared in test_cases - for (const auto& key : group_order) { - const auto& indices = groups[key]; - if (indices.empty()) - continue; - - // Get first test case as the "prototype" - size_t prototype_idx = indices[0]; - TestCase& prototype = test_cases[prototype_idx]; - - // Generate data for prototype with deterministic seed based on key - int group_seed = - static_cast(std::hash{}(key.key_string) % 10000); - for (auto& input : prototype.inputs()) { - input.ensure_data_generated(group_seed++); - } - - // Compute reference once for prototype - bool ref_computed = false; - if (reference_compute_func) { - try { - reference_compute_func(prototype); - ref_computed = true; - } catch (const std::invalid_argument&) { - // Reference computation skipped for this group - } - } - - // Copy data and reference to other test cases in group - for (size_t i = 1; i < indices.size(); ++i) { - size_t tc_idx = indices[i]; - TestCase& tc = test_cases[tc_idx]; - - // Copy input data from prototype - for (size_t j = 0; - j < tc.inputs().size() && j < prototype.inputs().size(); - ++j) { - auto& dest = tc.inputs()[j]; - const auto& src = prototype.inputs()[j]; - if (dest.is_tensor() && src.is_tensor() && dest.sizes == src.sizes && - dest.dtype == src.dtype) { - dest.share_data_from(src); - } - } - - // Copy reference output data if available - if (ref_computed) { - for (size_t j = 0; - j < tc.outputs().size() && j < prototype.outputs().size(); - ++j) { - const auto& src = prototype.outputs()[j]; - auto& dest = tc.outputs()[j]; - if (dest.is_tensor() && src.is_tensor() && dest.sizes == src.sizes && - dest.dtype == src.dtype) { - dest.share_reference_from(src); - } - } - } - } - - // Execute and print results for all test cases in this group - for (size_t tc_idx : indices) { - TestCase& test_case = test_cases[tc_idx]; - ++test_case_counter; - - // Execute single test case - BenchmarkResult result; - bool shader_not_supported = false; - try { - // Always run a probe pass at chained_dispatches=1 with - // write_outputs=true. This populates test_case.outputs() for the - // downstream correctness check with a clean single-dispatch result, - // and also gives us probe_us for adaptive sizing of the measurement - // run's chained_dispatches. - // - // probe_then_scale (Google Benchmark style): size chained_dispatches - // so each measurement execute() takes ~target_us. Tiny ops get a - // large factor (driving GPU governor escalation); heavy ops get a - // small factor (bounded wall time). Caveat: on Adreno 740 the probe - // runs at the DCVS-pinned clock (~220 MHz), so probe_us is inflated - // and the computed factor comes out under-sized for boost clock — but - // the default target is generous enough that an under-sized factor - // still drives sustained activity. - BenchmarkResult probe_result = execute_test_case( - test_case, - /*warmup_runs=*/1, - /*benchmark_runs=*/1, - /*chained_dispatches=*/1, - /*write_outputs=*/true); - float probe_us = probe_result.get_avg_time_us(); - if (probe_us < kMinProbeTimeUs) { - probe_us = kMinProbeTimeUs; - } - - int chained_dispatches; - const int manual_n = test_case.get_op_invocations_per_execute(); - if (manual_n > 0) { - chained_dispatches = manual_n; - } else { - const int target_us = test_case.get_target_execute_time_us(); - chained_dispatches = - std::max(1, static_cast(target_us / probe_us)); - chained_dispatches = - std::min(chained_dispatches, kMaxChainedDispatches); - } - if (debugging()) { - std::cout << "[probe] " << test_case.name() - << ": probe_us=" << probe_us - << ", chained_dispatches=" << chained_dispatches - << std::endl; - } - - // Measurement pass: chained_dispatches stacking, skip output copy - // since the probe already wrote the correctness data. - result = execute_test_case( - test_case, - warmup_runs, - benchmark_runs, - chained_dispatches, - /*write_outputs=*/false); - result.set_operator_name(test_case.operator_name()); - } catch (const vkcompute::vkapi::ShaderNotSupportedError&) { - result = BenchmarkResult( - test_case.name().empty() ? "unnamed_test_case" : test_case.name(), - test_case.operator_name()); - shader_not_supported = true; - } - - // Determine if this test case passed (has valid timing data) - bool vulkan_execute_succeeded = - result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; - - if (shader_not_supported) { - result.set_correctness_status(CorrectnessStatus::SKIPPED); - } else if (!vulkan_execute_succeeded) { - result.set_correctness_status(CorrectnessStatus::FAILED); - } else if (!ref_computed) { - result.set_correctness_status(CorrectnessStatus::SKIPPED); - } else { - // Reference function provided and succeeded - validate outputs - bool correctness_passed = true; - - for (size_t output_idx = 0; output_idx < test_case.num_outputs(); - ++output_idx) { - const ValueSpec& output_spec = test_case.outputs()[output_idx]; - - if (!output_spec.validate_against_reference( - test_case.get_abs_tolerance(), - test_case.get_rel_tolerance())) { - correctness_passed = false; - std::cout << " Correctness validation FAILED for test " - << result.get_kernel_name() << std::endl; - print_valuespec_data(output_spec, "vulkan output"); - print_valuespec_data(output_spec, "ref output", true); - } - } - - if (correctness_passed) { - result.set_correctness_status(CorrectnessStatus::PASSED); - } else { - any_correctness_failed = true; - result.set_correctness_status(CorrectnessStatus::FAILED); - } - } - - // Calculate GFLOPS for this test case using the provided FLOP calculator - float case_gflops = 0.0f; - if (vulkan_execute_succeeded) { - // Use the provided FLOP calculator to get total FLOPs for this test - // case - int64_t total_flops = flop_calculator(test_case); - float flops = static_cast(total_flops); - float avg_time_us = result.get_avg_time_us(); - if (avg_time_us > 0.0f && total_flops > 0) { - case_gflops = (flops / 1e9f) / (avg_time_us / 1e6f); - } - - total_gflops += case_gflops; - } else { - case_gflops = -1.0f; // Indicate failure - } - - // Calculate tensor info for display - std::string size_info = "["; - if (!test_case.empty() && test_case.num_inputs() > 0 && - test_case.inputs()[0].is_tensor()) { - const auto& sizes = test_case.inputs()[0].get_tensor_sizes(); - for (size_t j = 0; j < sizes.size(); ++j) { - size_info += std::to_string(sizes[j]); - if (j < sizes.size() - 1) - size_info += "x"; - } - } - size_info += "]"; - - // Print progress using the BenchmarkResult member function - result.print_summary(test_case_counter, size_info, case_gflops); - - // Add result to collection - results.add_result(std::move(result)); - - test_case.clear(); - } - } - - // Set the overall results on the TestResult - results.set_correctness_passed(!any_correctness_failed); - results.set_gflops(total_gflops); - - print_separator(); - std::cout << "Completed " << results.size() << " test cases" << std::endl; - - // Hard-fail if any correctness check failed. The per-case loop above keeps - // running after a mismatch (so the full pass/fail matrix and per-16x16-tile - // mismatch maps are printed for debugging) rather than aborting on the first - // failure; throwing here preserves the original abort-on-mismatch behavior. - if (any_correctness_failed) { - throw std::runtime_error("Correctness validation failed"); - } - - return results; -} - -// Convenience overload that uses the default FLOP calculator -TestResult execute_test_cases( - std::function()> test_case_generator, - const std::string& operation_name, - int warmup_runs, - int benchmark_runs, - ReferenceComputeFunc reference_compute_func) { - return execute_test_cases( - test_case_generator, - default_flop_calculator, - operation_name, - warmup_runs, - benchmark_runs, - reference_compute_func); -} - -// Utility functions for printing -void print_performance_header() { - std::cout << "\n=== Compute Shader Performance Benchmark ===" << std::endl; -} - -void print_separator() { - std::cout << std::string(70, '-') << std::endl; -} - -// ValueSpec data printing utilities -void print_valuespec_data( - const ValueSpec& spec, - const std::string& name, - const bool print_ref_data, - size_t max_elements, - int precision) { - std::cout << "\n" << name << " Data:" << std::endl; - std::cout << " Type: " << spec.to_string() << std::endl; - - if (!spec.is_tensor()) { - if (spec.is_int()) { - std::cout << " Value: " << spec.get_int_value() << std::endl; - } else if (spec.is_int_list()) { - const auto& int_list = spec.get_int_list(); - std::cout << " Values: ["; - size_t print_count = std::min(max_elements, int_list.size()); - for (size_t i = 0; i < print_count; ++i) { - std::cout << int_list[i]; - if (i < print_count - 1) - std::cout << ", "; - } - if (int_list.size() > max_elements) { - std::cout << ", ... (" << (int_list.size() - max_elements) << " more)"; - } - std::cout << "]" << std::endl; - } - return; - } - - // Print tensor data - size_t total_elements = spec.numel(); - size_t print_count = std::min(max_elements, total_elements); - - std::cout << " Total elements: " << total_elements << std::endl; - std::cout << " Data (first " << print_count << " elements): ["; - - std::cout << std::fixed << std::setprecision(precision); - - switch (spec.dtype) { - case vkapi::kFloat: { - auto data = spec.get_float_data().data(); - if (print_ref_data) { - data = spec.get_ref_float_data().data(); - } - for (size_t i = 0; i < print_count; ++i) { - std::cout << data[i]; - if (i < print_count - 1) - std::cout << ", "; - } - break; - } - case vkapi::kHalf: { - if (print_ref_data) { - const auto& ref = spec.get_ref_float_data(); - for (size_t i = 0; i < print_count; ++i) { - std::cout << ref[i]; - if (i < print_count - 1) - std::cout << ", "; - } - break; - } - const auto& data = spec.get_half_data(); - for (size_t i = 0; i < print_count; ++i) { - // Convert IEEE 754 half-precision bit pattern back to float. - float value = half_to_float(data[i]); - std::cout << value; - if (i < print_count - 1) - std::cout << ", "; - } - break; - } - case vkapi::kInt: { - const auto& data = spec.get_int32_data(); - for (size_t i = 0; i < print_count; ++i) { - std::cout << data[i]; - if (i < print_count - 1) - std::cout << ", "; - } - break; - } - case vkapi::kChar: { - const auto& data = spec.get_int8_data(); - if (spec.is_int4()) { - // Print each 4-bit value individually - size_t element_count = 0; - for (size_t i = 0; i < data.size() && element_count < print_count; - ++i) { - // Extract lower 4 bits (signed) - int8_t lower_4bits = data[i] & 0x0F; - if (lower_4bits > 7) - lower_4bits -= 16; // Convert to signed - std::cout << static_cast(lower_4bits); - element_count++; - - if (element_count < print_count) { - std::cout << ", "; - // Extract upper 4 bits (signed) - int8_t upper_4bits = (data[i] >> 4) & 0x0F; - if (upper_4bits > 7) - upper_4bits -= 16; // Convert to signed - std::cout << static_cast(upper_4bits); - element_count++; - - if (element_count < print_count) - std::cout << ", "; - } - } - } else { - for (size_t i = 0; i < print_count; ++i) { - std::cout << static_cast(data[i]); - if (i < print_count - 1) - std::cout << ", "; - } - } - break; - } - case vkapi::kByte: { - const auto& data = spec.get_uint8_data(); - if (spec.is_int4()) { - // Print each 4-bit value individually - size_t element_count = 0; - for (size_t i = 0; i < data.size() && element_count < print_count; - ++i) { - // Extract lower 4 bits - uint8_t lower_4bits = data[i] & 0x0F; - std::cout << static_cast(lower_4bits); - element_count++; - - if (element_count < print_count) { - std::cout << ", "; - // Extract upper 4 bits - uint8_t upper_4bits = (data[i] >> 4) & 0x0F; - std::cout << static_cast(upper_4bits); - element_count++; - - if (element_count < print_count) - std::cout << ", "; - } - } - } else { - for (size_t i = 0; i < print_count; ++i) { - std::cout << static_cast(data[i]); - if (i < print_count - 1) - std::cout << ", "; - } - } - break; - } - default: - std::cout << "unsupported data type"; - break; - } - - if (total_elements > max_elements) { - std::cout << ", ... (" << (total_elements - max_elements) << " more)"; - } - std::cout << "]" << std::endl; - - // Print some statistics for tensor data - if (total_elements > 0) { - float min_val = 0.0f, max_val = 0.0f, sum = 0.0f; - bool first = true; - - for (size_t i = 0; i < total_elements; ++i) { - float val = spec.get_element(i); - if (first) { - min_val = max_val = val; - first = false; - } else { - min_val = std::min(min_val, val); - max_val = std::max(max_val, val); - } - sum += val; - } - - float mean = sum / total_elements; - std::cout << " Statistics: min=" << std::setprecision(precision) << min_val - << ", max=" << max_val << ", mean=" << mean << ", sum=" << sum - << std::endl; - } -} - -ValueRef quantized_weights_canvas( - ComputeGraph& graph, - const ValueRef weight_ref) { - const auto original_sizes = graph.sizes_of(weight_ref); - - std::vector sorted_sizes = original_sizes; - std::sort(sorted_sizes.begin(), sorted_sizes.end(), std::greater()); - int64_t largest1 = sorted_sizes.size() > 0 ? sorted_sizes[0] : 0; - - std::vector final_sizes = {1, largest1, largest1}; - - // Debug logging if debugging flag is set - if (debugging()) { - std::cout << "Debug: Creating quantized weights canvas tensor" << std::endl; - std::cout << "Debug: Original sizes: ["; - for (size_t i = 0; i < original_sizes.size(); ++i) { - std::cout << original_sizes[i]; - if (i < original_sizes.size() - 1) - std::cout << ", "; - } - std::cout << "]" << std::endl; - std::cout << "Debug: Canvas sizes: ["; - for (size_t i = 0; i < final_sizes.size(); ++i) { - std::cout << final_sizes[i]; - if (i < final_sizes.size() - 1) - std::cout << ", "; - } - std::cout << "]" << std::endl; - } - - ValueRef packed_weight = graph.add_tensor( - final_sizes, vkapi::kInt, utils::kTexture3D, utils::kWidthPacked); - - std::string kernel_name = "packed_int32_canvas"; - add_storage_type_suffix(kernel_name, graph.storage_type_of(packed_weight)); - - graph.prepack_nodes().emplace_back(new PrepackNode( - graph, - VK_KERNEL_FROM_STR(kernel_name), - graph.create_gwg(packed_weight), - graph.create_lwg(packed_weight), - weight_ref, - packed_weight, - // UBOs - {graph.logical_limits_ubo(packed_weight)}, - // Specialization constants - {}, - // Push Constants - {})); - - return packed_weight; -} - -ValueRef float_tensor_canvas(ComputeGraph& graph, const ValueRef weight_ref) { - const auto original_sizes = graph.sizes_of(weight_ref); - - std::vector sorted_sizes = original_sizes; - std::sort(sorted_sizes.begin(), sorted_sizes.end(), std::greater()); - int64_t largest1 = sorted_sizes.size() > 0 ? sorted_sizes[0] : 0; - - std::vector final_sizes = {1, largest1, largest1}; - - // Debug logging if debugging flag is set - if (debugging()) { - std::cout << "Debug: Creating float tensor canvas" << std::endl; - std::cout << "Debug: Original sizes: ["; - for (size_t i = 0; i < original_sizes.size(); ++i) { - std::cout << original_sizes[i]; - if (i < original_sizes.size() - 1) - std::cout << ", "; - } - std::cout << "]" << std::endl; - std::cout << "Debug: Canvas sizes: ["; - for (size_t i = 0; i < final_sizes.size(); ++i) { - std::cout << final_sizes[i]; - if (i < final_sizes.size() - 1) - std::cout << ", "; - } - std::cout << "]" << std::endl; - } - - ValueRef packed_weight = graph.add_tensor( - final_sizes, vkapi::kFloat, utils::kTexture3D, utils::kWidthPacked); - - graph.prepack_nodes().emplace_back(new PrepackNode( - graph, - VK_KERNEL_FROM_STR("float_canvas"), - graph.create_gwg(packed_weight), - graph.create_lwg(packed_weight), - weight_ref, - packed_weight, - // UBOs - {graph.logical_limits_ubo(packed_weight)}, - // Specialization constants - {}, - // Push Constants - {})); - - return packed_weight; -} - -// Compute weight sums for quantized operations (linear and convolution) -void compute_weight_sums( - ValueSpec& weight_sums, - const ValueSpec& quantized_weight, - int64_t out_features, - int64_t elements_per_output_feature) { - auto& weight_sums_data = weight_sums.get_int32_data(); - auto& quantized_weight_data = quantized_weight.get_int8_data(); - - // Don't resize down - the buffer may be pre-allocated with aligned size. - // Only resize up if needed. - if (weight_sums_data.size() < static_cast(out_features)) { - weight_sums_data.resize(out_features); - } - - // For each output feature, compute the sum of quantized weights - for (int64_t out_f = 0; out_f < out_features; ++out_f) { - int32_t sum = 0; - for (int64_t elem = 0; elem < elements_per_output_feature; ++elem) { - // Weight indexing depends on the layout: - // For linear: [out_features, in_features] -> out_f * - // elements_per_output_feature + elem For conv2d: [C_out, C_in * K_h * - // K_w] -> out_f * elements_per_output_feature + elem - int64_t weight_idx = out_f * elements_per_output_feature + elem; - sum += static_cast(quantized_weight_data[weight_idx]); - } - weight_sums_data[out_f] = sum; - } -} - -// Compute weight sums for 4D quantized conv2d operations -// Weight layout: [C_out, K_h, K_w, align_up_4(C_in_per_group)] -void compute_weight_sums_4d( - ValueSpec& weight_sums, - const ValueSpec& quantized_weight, - int64_t out_channels, - int64_t kernel_h, - int64_t kernel_w, - int64_t aligned_in_channels) { - auto& weight_sums_data = weight_sums.get_int32_data(); - auto& quantized_weight_data = quantized_weight.get_int8_data(); - - weight_sums_data.resize(out_channels); - - // For each output channel, compute the sum of quantized weights - for (int64_t out_c = 0; out_c < out_channels; ++out_c) { - int32_t sum = 0; - - for (int64_t kh = 0; kh < kernel_h; ++kh) { - for (int64_t kw = 0; kw < kernel_w; ++kw) { - for (int64_t in_c = 0; in_c < aligned_in_channels; ++in_c) { - // Weight indexing: [out_c, kh, kw, in_c] - int64_t weight_idx = - out_c * (kernel_h * kernel_w * aligned_in_channels) + - kh * (kernel_w * aligned_in_channels) + kw * aligned_in_channels + - in_c; - sum += static_cast(quantized_weight_data[weight_idx]); - } - } - } - - weight_sums_data[out_c] = sum; - } -} - -// Helper function to unpack 4-bit values from uint8 (same as in -// q4gsw_linear.cpp) -std::pair unpack_4bit_utils(uint8_t packed) { - // Extract lower 4 bits and upper 4 bits - int8_t lower = packed & 0x0F; - int8_t upper = (packed >> 4) & 0x0F; - - // Subtract 8 from unpacked 4-bit values - lower -= 8; - upper -= 8; - - return std::make_pair(lower, upper); -} - -// Compute weight sums for 4-bit group symmetric quantized weights -void compute_weight_sums_4bit_grouped( - ValueSpec& weight_sums, - const ValueSpec& quantized_weight, - int64_t num_groups, - int64_t out_features, - int64_t group_size) { - auto& weight_sums_data = weight_sums.get_int32_data(); - auto& quantized_weight_data = quantized_weight.get_uint8_data(); - - // Resize to [num_groups, out_features] - weight_sums_data.resize(num_groups * out_features); - - // For each group and each output feature, compute the sum of quantized - // weights in that group - for (int64_t group_idx = 0; group_idx < num_groups; ++group_idx) { - for (int64_t out_f = 0; out_f < out_features; ++out_f) { - int32_t sum = 0; - - // Sum weights for this group and output feature - for (int64_t in_group = 0; in_group < group_size; ++in_group) { - int64_t in_f = group_idx * group_size + in_group; - - // Get packed weight value - weight matrix is [N, K/2] - int64_t weight_idx = - out_f * ((num_groups * group_size) / 2) + (in_f / 2); - uint8_t packed_weight = quantized_weight_data[weight_idx]; - - // Unpack 4-bit weight - auto unpacked = unpack_4bit_utils(packed_weight); - int8_t weight_4bit = (in_f % 2 == 0) ? unpacked.first : unpacked.second; - - sum += static_cast(weight_4bit); - } - - // Store sum for this group and output feature - int64_t sums_idx = group_idx * out_features + out_f; - weight_sums_data[sums_idx] = sum; - } - } -} - -} // namespace prototyping -} // namespace vulkan -} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/utils.h b/backends/vulkan/test/custom_ops/utils.h deleted file mode 100644 index b94d0d41421..00000000000 --- a/backends/vulkan/test/custom_ops/utils.h +++ /dev/null @@ -1,1094 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. -// All rights reserved. -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#pragma once - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace executorch { -namespace vulkan { -namespace prototyping { - -using namespace vkcompute; - -// -// ReferenceKey for caching reference computations -// - -// Captures the identity of input conditions for test case grouping. -// Test cases with the same ReferenceKey should produce identical reference -// outputs, so reference computation can be cached and reused. -struct ReferenceKey { - std::string key_string; - - static ReferenceKey from_test_case(const class TestCase& tc); - - bool operator==(const ReferenceKey& other) const { - return key_string == other.key_string; - } -}; - -struct ReferenceKeyHash { - size_t operator()(const ReferenceKey& k) const { - return std::hash{}(k.key_string); - } -}; - -// -// Global configuration options -// - -bool print_output(); -void set_print_output(bool print_output); - -bool print_latencies(); -void set_print_latencies(bool print_latencies); - -bool use_gpu_timestamps(); -void set_use_gpu_timestamps(bool use_timestamps); - -bool debugging(); -void set_debugging(bool enable_debugging); - -// -// Shader filter defaults -// - -// Default shader filter: excludes layout conversions and quantization overhead -// This is used for tests where quantize/dequantize are overhead operations -inline const std::vector kDefaultShaderFilter = { - "nchw_to", - "to_nchw", - "quantize_and_pack_4w4c", - "unpack_4w4c_and_dequantize"}; - -// Layout-only filter: only excludes layout conversions -// Use this for tests where quantize/dequantize ARE the operations being tested -inline const std::vector kLayoutOnlyShaderFilter = { - "nchw_to", - "to_nchw"}; - -// -// String utilities -// - -// Helper function to get abbreviated layout names for test case naming -inline std::string layout_abbrev(utils::GPUMemoryLayout layout) { - switch (layout) { - case utils::kWidthPacked: - return "WP"; - case utils::kChannelsPacked: - return "CP"; - case utils::kPackedInt8_4W: - return "4W"; - case utils::kPackedInt8_4C: - return "4C"; - case utils::kPackedInt8_4W4C: - return "4W4C"; - case utils::kPackedInt8_4H4W: - return "4H4W"; - case utils::kPackedInt8_4C1W: - return "4C1W"; - default: - return "UNK"; - } -} - -// Helper function to get abbreviated storage type names for test case naming -inline std::string storage_type_abbrev(utils::StorageType storage_type) { - switch (storage_type) { - case utils::kTexture3D: - return "Tex"; - case utils::kBuffer: - return "Buf"; - default: - return "UNK"; - } -} - -// Helper function to get combined storage type and layout representation -// Example: (kBuffer, kPackedInt8_4W4C) -> "Buf_4W4C" -inline std::string repr_str( - utils::StorageType storage_type, - utils::GPUMemoryLayout layout) { - return storage_type_abbrev(storage_type) + "(" + layout_abbrev(layout) + ")"; -} - -// Helper function to generate comma-separated shape string for test case naming -// Example: {1, 128, 56, 56} -> "1,128,56,56" -inline std::string shape_string(const std::vector& shape) { - std::string result; - for (size_t i = 0; i < shape.size(); ++i) { - if (i > 0) { - result += ","; - } - result += std::to_string(shape[i]); - } - return result; -} - -// Helper function to generate a bracketed shape string for test case naming -// Example: {1, 128, 56, 56} -> "[1,128,56,56]" -inline std::string shape_bracket(const std::vector& shape) { - return "[" + shape_string(shape) + "]"; -} - -// Short dtype symbol used in standardized test case labels -// Example: kFloat -> "f32", kHalf -> "f16", kChar -> "i8", kInt -> "i32" -inline std::string dtype_short(vkapi::ScalarType dtype) { - switch (dtype) { - case vkapi::kFloat: - return "f32"; - case vkapi::kHalf: - return "f16"; - case vkapi::kChar: - return "i8"; - case vkapi::kByte: - return "u8"; - case vkapi::kInt: - return "i32"; - case vkapi::kBool: - return "b"; - default: - return "?"; - } -} - -// Build a standardized test label of the form: -// " [ ]" -// where is "->" when the two differ, or just -// "" when they match. Sections are separated by two spaces. If -// suffix is non-empty it is appended after a single space (allowing callers -// to pass e.g. "[general]" or "[gemv] +bias"). -inline std::string make_test_label( - const std::string& prefix, - const std::string& in_dtype, - const std::string& out_dtype, - const std::string& shape_str, - const std::string& storage_str, - const std::string& suffix = "") { - const std::string dtype_str = - (in_dtype == out_dtype) ? in_dtype : in_dtype + "->" + out_dtype; - std::string label = - prefix + " " + dtype_str + " " + shape_str + " " + storage_str; - if (!suffix.empty()) { - label += " " + suffix; - } - return label; -} - -// -// ValueSpec class -// - -enum class SpecType { Tensor, IntList, Int, Float, Bool, String }; - -// Data generation types -enum class DataGenType { - FIXED, - RANDOM, - RANDOM_SCALES, - RANDINT, - RANDINT8, - RANDINT4, - ONES, - ONES_INT4, - ZEROS -}; - -// Value specification struct -struct ValueSpec { - std::vector sizes; - vkapi::ScalarType dtype; - utils::GPUMemoryLayout memory_layout; - utils::StorageType storage_type; - SpecType spec_type; - DataGenType data_gen_type; - bool is_constant_tensor; - bool is_none_flag; - bool is_int4_tensor; - std::string string_data; - - ValueSpec( - const std::vector& sizes, - vkapi::ScalarType dtype, - utils::StorageType storage_type = utils::kTexture3D, - utils::GPUMemoryLayout memory_layout = utils::kWidthPacked) - : sizes(sizes), - dtype(dtype), - memory_layout(memory_layout), - storage_type(storage_type), - spec_type(SpecType::Tensor), - data_gen_type(DataGenType::ZEROS), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(false) { - // Data generation is deferred until first access (any data getter or - // ensure_data_generated() triggers it). - } - - // Constructor for tensor with custom data generation type - ValueSpec( - const std::vector& sizes, - vkapi::ScalarType dtype, - utils::StorageType storage_type, - utils::GPUMemoryLayout memory_layout, - DataGenType data_gen_type) - : sizes(sizes), - dtype(dtype), - memory_layout(memory_layout), - storage_type(storage_type), - spec_type(SpecType::Tensor), - data_gen_type(data_gen_type), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(false) { - // Data generation is deferred until first access (any data getter or - // ensure_data_generated() triggers it). - } - - // Constructor for single int - ValueSpec(int32_t value) - : sizes({1}), - dtype(vkapi::kInt), - memory_layout(utils::kWidthPacked), - storage_type(utils::kTexture3D), - spec_type(SpecType::Int), - data_gen_type(DataGenType::FIXED), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(true) { - data_->int32_data.push_back(value); - } - - // Constructor for single float - ValueSpec(float value) - : sizes({1}), - dtype(vkapi::kFloat), - memory_layout(utils::kWidthPacked), - storage_type(utils::kTexture3D), - spec_type(SpecType::Float), - data_gen_type(DataGenType::FIXED), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(true) { - data_->float_data.push_back(value); - } - - // Constructor for single bool - ValueSpec(bool value) - : sizes({1}), - dtype(vkapi::kInt), - memory_layout(utils::kWidthPacked), - storage_type(utils::kTexture3D), - spec_type(SpecType::Bool), - data_gen_type(DataGenType::FIXED), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(true) { - data_->int32_data.push_back(value ? 1 : 0); - } - - // Constructor for int list - ValueSpec(const std::vector& values) - : sizes({static_cast(values.size())}), - dtype(vkapi::kInt), - memory_layout(utils::kWidthPacked), - storage_type(utils::kTexture3D), - spec_type(SpecType::IntList), - data_gen_type(DataGenType::FIXED), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(true) { - data_->int32_data = values; - } - - // Factory method for string (avoids ambiguity with vector constructor) - static ValueSpec make_string(const std::string& value) { - ValueSpec spec; - spec.sizes = {1}; - spec.dtype = vkapi::kInt; - spec.memory_layout = utils::kWidthPacked; - spec.storage_type = utils::kTexture3D; - spec.spec_type = SpecType::String; - spec.data_gen_type = DataGenType::FIXED; - spec.is_constant_tensor = false; - spec.is_none_flag = false; - spec.is_int4_tensor = false; - spec.data_generated_ = true; - spec.string_data = value; - return spec; - } - - // Default constructor - ValueSpec() - : dtype(vkapi::kFloat), - memory_layout(utils::kWidthPacked), - storage_type(utils::kTexture3D), - spec_type(SpecType::Tensor), - data_gen_type(DataGenType::ZEROS), - is_constant_tensor(false), - is_none_flag(false), - is_int4_tensor(false), - data_generated_(false) {} - - int64_t numel() const; - size_t nbytes() const; - std::string to_string() const; - - bool is_tensor() const { - return spec_type == SpecType::Tensor; - } - bool is_int_list() const { - return spec_type == SpecType::IntList; - } - bool is_int() const { - return spec_type == SpecType::Int; - } - bool is_float() const { - return spec_type == SpecType::Float; - } - bool is_bool() const { - return spec_type == SpecType::Bool; - } - bool is_string() const { - return spec_type == SpecType::String; - } - - int32_t get_int_value() const { - ensure_data_generated(); - return data_->int32_data.empty() ? 0 : data_->int32_data[0]; - } - float get_float_value() const { - ensure_data_generated(); - return data_->float_data.empty() ? 0.0f : data_->float_data[0]; - } - bool get_bool_value() const { - ensure_data_generated(); - return data_->int32_data.empty() ? false : (data_->int32_data[0] != 0); - } - const std::string& get_string_value() const { - return string_data; - } - const std::vector& get_int_list() const { - ensure_data_generated(); - return data_->int32_data; - } - const std::vector& get_tensor_sizes() const { - return sizes; - } - - // References and pointers into tensor data must not be held across any other - // access to the same spec: a mutable access may detach the shared payload, - // leaving a previously returned reference bound to the old payload. Consume - // immediately. - const std::vector& get_float_data() const { - ensure_data_generated(); - return data_->float_data; - } - const std::vector& get_int32_data() const { - ensure_data_generated(); - return data_->int32_data; - } - const std::vector& get_half_data() const { - ensure_data_generated(); - return data_->half_data; - } - const std::vector& get_int8_data() const { - ensure_data_generated(); - return data_->int8_data; - } - const std::vector& get_uint8_data() const { - ensure_data_generated(); - return data_->uint8_data; - } - - std::vector& get_float_data() { - ensure_data_generated(); - ensure_unique_data(); - return data_->float_data; - } - std::vector& get_int32_data() { - ensure_data_generated(); - ensure_unique_data(); - return data_->int32_data; - } - std::vector& get_half_data() { - ensure_data_generated(); - ensure_unique_data(); - return data_->half_data; - } - std::vector& get_int8_data() { - ensure_data_generated(); - ensure_unique_data(); - return data_->int8_data; - } - std::vector& get_uint8_data() { - ensure_data_generated(); - ensure_unique_data(); - return data_->uint8_data; - } - - const std::vector& get_ref_float_data() const { - return reference_data_->float_data; - } - const std::vector& get_ref_int32_data() const { - return reference_data_->int32_data; - } - const std::vector& get_ref_half_data() const { - return reference_data_->half_data; - } - const std::vector& get_ref_int8_data() const { - return reference_data_->int8_data; - } - const std::vector& get_ref_uint8_data() const { - return reference_data_->uint8_data; - } - - std::vector& get_ref_float_data() { - ensure_unique_reference_data(); - return reference_data_->float_data; - } - std::vector& get_ref_int32_data() { - ensure_unique_reference_data(); - return reference_data_->int32_data; - } - std::vector& get_ref_half_data() { - ensure_unique_reference_data(); - return reference_data_->half_data; - } - std::vector& get_ref_int8_data() { - ensure_unique_reference_data(); - return reference_data_->int8_data; - } - std::vector& get_ref_uint8_data() { - ensure_unique_reference_data(); - return reference_data_->uint8_data; - } - - void resize_data(size_t new_size); - void* get_mutable_data_ptr(); - float get_element(size_t index) const; - - // Data generation methods for deferred generation and caching. - // - // ValueSpec is not thread-safe: lazy materialization and copy-on-write - // detach mutate shared state from const methods. Test cases are built and - // executed on a single thread. - // - // Implicit materialization (any data getter, resize_data) consumes the - // global seed counter. Callers needing deterministic data must call - // ensure_data_generated(explicit_seed) before any other access; a later - // seeded call is a no-op once data is generated. - bool is_data_generated() const { - return data_generated_; - } - void ensure_data_generated(int seed = -1) const; - void share_data_from(const ValueSpec& other); - void share_reference_from(const ValueSpec& other); - - // Set/get constant flag - bool is_constant() const { - return is_constant_tensor; - } - void set_constant(bool is_constant) { - is_constant_tensor = is_constant; - } - - // Set/get none flag - bool is_none() const { - return is_none_flag; - } - - void set_none(bool is_none) { - is_none_flag = is_none; - } - - // Set/get int4 flag - bool is_int4() const { - return is_int4_tensor; - } - void set_int4(bool is_int4) { - is_int4_tensor = is_int4; - } - - const void* get_data_ptr() const; - - // Correctness checking against reference data - // Returns true if computed data matches reference data within tolerance - // Only validates float tensors as specified in requirements - bool validate_against_reference( - float abs_tolerance = 2e-3f, - float rel_tolerance = 1e-3f) const; - - private: - struct TensorData { - std::vector float_data; - std::vector int32_data; - std::vector half_data; - std::vector int8_data; - std::vector uint8_data; - }; - - void ensure_unique_data() const; - void ensure_unique_reference_data() const; - void generate_tensor_data(int seed = -1) const; - - mutable bool data_generated_ = false; - mutable std::shared_ptr data_ = std::make_shared(); - mutable std::shared_ptr reference_data_ = - std::make_shared(); -}; - -// -// TestCase -// - -// Default per-execute() wall-clock target used by the probe-then-scale logic -// in execute_test_cases(). Picked generously enough that even an under-sized -// chained_dispatches factor (the probe runs at governor-pinned clock and -// underestimates the boost-clock latency) still drives sustained GPU -// activity during measurement. -constexpr int kDefaultTargetExecuteTimeUs = 100000; - -class TestCase { - public: - TestCase() - : abs_tolerance_(2e-3f), - rel_tolerance_(1e-3f), - shader_filter_(kDefaultShaderFilter) {} - TestCase(const std::string& name) - : name_(name), - abs_tolerance_(2e-3f), - rel_tolerance_(1e-3f), - shader_filter_(kDefaultShaderFilter) {} - - void set_name(const std::string& name) { - name_ = name; - } - const std::string& name() const { - return name_; - } - - void set_operator_name(const std::string& op_name) { - operator_name_ = op_name; - } - const std::string& operator_name() const { - return operator_name_; - } - - // Tolerance settings - void set_abs_tolerance(float abs_tolerance) { - abs_tolerance_ = abs_tolerance; - } - float get_abs_tolerance() const { - return abs_tolerance_; - } - - void set_rel_tolerance(float rel_tolerance) { - rel_tolerance_ = rel_tolerance; - } - float get_rel_tolerance() const { - return rel_tolerance_; - } - - // Shader filter settings - list of shader name patterns to exclude from - // timing - void set_shader_filter(const std::vector& filter) { - shader_filter_ = filter; - } - const std::vector& get_shader_filter() const { - return shader_filter_; - } - - // Manual override for the number of chained dispatches per measurement - // iteration (a.k.a. chained_dispatches). If > 0, the framework uses this - // directly and skips the probe phase. 0 (the default) means adaptive - // (probe-then-scale). - void set_op_invocations_per_execute(int n) { - op_invocations_per_execute_ = n; - } - int get_op_invocations_per_execute() const { - return op_invocations_per_execute_; - } - - // Target single-execute duration in microseconds. Used only when the - // manual chained_dispatches override is not set. Default - // kDefaultTargetExecuteTimeUs, picked generously enough to mitigate Adreno - // DCVS governor pinning during the probe. - void set_target_execute_time_us(int us) { - target_execute_time_us_ = us; - } - int get_target_execute_time_us() const { - return target_execute_time_us_; - } - - // When true, the ComputeGraph built for this test case sets - // GraphConfig::force_resize, so every DynamicDispatchNode runs its resize - // function once during measurement setup (execute_test_case runs - // propagate_resize() after prepack) even when no input shape changed. - // Because the output is already allocated at the swept shape, the resize - // must recompute the same shape from the current input — a wrong resize - // formula resizes the output to a mismatched shape and surfaces as a test - // failure. Default true (opt-out): every custom_ops test exercises its - // resize formulas across the swept shapes. Call set_force_resize(false) for - // the rare op whose resize fn is intentionally not shape-preserving under a - // fixed output allocation. - void set_force_resize(bool force_resize) { - force_resize_ = force_resize; - } - bool get_force_resize() const { - return force_resize_; - } - - // When true, the ComputeGraph sets GraphConfig::force_narrow_int4_tile, so - // the int4 linear picks its narrow (2-row) output tile on any GPU. That - // variant is otherwise reachable only on Mali, which no CI runner has. - void set_force_narrow_int4_tile(bool force) { - force_narrow_int4_tile_ = force; - } - bool get_force_narrow_int4_tile() const { - return force_narrow_int4_tile_; - } - - void add_input_spec(const ValueSpec& spec) { - inputs_.push_back(spec); - } - - const std::vector& inputs() const { - return inputs_; - } - - std::vector& inputs() { - return inputs_; - } - - size_t num_inputs() const { - return inputs_.size(); - } - - void add_output_spec(const ValueSpec& spec) { - outputs_.push_back(spec); - } - - const std::vector& outputs() const { - return outputs_; - } - - std::vector& outputs() { - return outputs_; - } - - size_t num_outputs() const { - return outputs_.size(); - } - - bool empty() const { - return inputs_.empty() && outputs_.empty(); - } - void clear() { - inputs_.clear(); - outputs_.clear(); - name_.clear(); - operator_name_.clear(); - abs_tolerance_ = 2e-3f; - rel_tolerance_ = 1e-3f; - shader_filter_ = kDefaultShaderFilter; - op_invocations_per_execute_ = 0; - target_execute_time_us_ = kDefaultTargetExecuteTimeUs; - force_resize_ = true; - force_narrow_int4_tile_ = false; - } - - private: - std::string name_; - std::string operator_name_; - std::vector inputs_; - std::vector outputs_; - float abs_tolerance_; - float rel_tolerance_; - std::vector shader_filter_; - int op_invocations_per_execute_ = 0; // 0 = adaptive - int target_execute_time_us_ = kDefaultTargetExecuteTimeUs; - bool force_resize_ = true; - bool force_narrow_int4_tile_ = false; -}; - -// -// BenchmarkResult -// - -enum class CorrectnessStatus { - SKIPPED, // No reference function provided - PASSED, // Reference function provided and validation passed - FAILED // Reference function provided but validation failed -}; - -// Per-shader timing data for detailed reporting -struct ShaderTiming { - std::string shader_name; - std::vector iter_timings_us; // Individual iteration timings - uint32_t gwg[3] = {0, 0, 0}; - uint32_t lwg[3] = {0, 0, 0}; - - float get_avg_time_us() const { - if (iter_timings_us.empty()) { - return 0.0f; - } - float sum = 0.0f; - for (float t : iter_timings_us) { - sum += t; - } - return sum / iter_timings_us.size(); - } -}; - -class BenchmarkResult { - public: - BenchmarkResult() : correctness_status_(CorrectnessStatus::SKIPPED) {} - - BenchmarkResult(const std::string& name) - : kernel_name(name), correctness_status_(CorrectnessStatus::SKIPPED) {} - - BenchmarkResult( - const std::string& kernel_name, - const std::string& operator_name) - : kernel_name(kernel_name), - operator_name(operator_name), - correctness_status_(CorrectnessStatus::SKIPPED) {} - - // Add timing for a single iteration - void add_iter_timing(float time_us); - - // Add per-shader timing for a single iteration - void add_shader_timing( - const std::string& shader_name, - float time_us, - const uint32_t gwg[3], - const uint32_t lwg[3]); - - // Get per-shader timing data - const std::vector& get_shader_timings() const { - return shader_timings_; - } - - // Getters - const std::string& get_kernel_name() const { - return kernel_name; - } - const std::string& get_operator_name() const { - return operator_name; - } - float get_avg_time_us() const; - size_t get_num_iterations() const { - return iter_timings.size(); - } - const std::vector& get_iter_timings() const { - return iter_timings; - } - CorrectnessStatus get_correctness_status() const { - return correctness_status_; - } - - // Setters - void set_kernel_name(const std::string& name) { - kernel_name = name; - } - void set_operator_name(const std::string& name) { - operator_name = name; - } - void set_correctness_status(CorrectnessStatus status) { - correctness_status_ = status; - } - - // Statistics - float get_min_time_us() const; - float get_max_time_us() const; - float get_std_dev_us() const; - - // Clear all timings - void clear_timings() { - iter_timings.clear(); - } - - // Print progress for this benchmark result - void print_summary( - int case_number, - const std::string& size_info, - float total_gflops) const; - - private: - std::string kernel_name; - std::string operator_name; - std::vector - iter_timings; // Individual iteration timings in microseconds - std::vector shader_timings_; // Per-shader timing data - CorrectnessStatus correctness_status_; -}; - -// Test result collection and processing -class TestResult { - public: - TestResult() : gflops_(0.0f), correctness_passed_(true) {} - TestResult(const std::string& operation_name) - : operation_name_(operation_name), - gflops_(0.0f), - correctness_passed_(true) {} - - // Add a benchmark result - void add_result(const BenchmarkResult& result); - void add_result(BenchmarkResult&& result); - - // Getters - const std::string& get_operation_name() const { - return operation_name_; - } - float get_gflops() const { - return gflops_; - } - bool get_correctness_passed() const { - return correctness_passed_; - } - size_t size() const { - return results_.size(); - } - bool empty() const { - return results_.empty(); - } - - // Setters - void set_gflops(float gflops_val) { - gflops_ = gflops_val; - } - void set_correctness_passed(bool passed) { - correctness_passed_ = passed; - } - - // Access results - const BenchmarkResult& operator[](size_t index) const { - return results_[index]; - } - BenchmarkResult& operator[](size_t index) { - return results_[index]; - } - const std::vector& get_results() const { - return results_; - } - - // Iterator support - std::vector::iterator begin() { - return results_.begin(); - } - std::vector::iterator end() { - return results_.end(); - } - std::vector::const_iterator begin() const { - return results_.begin(); - } - std::vector::const_iterator end() const { - return results_.end(); - } - - // Processing and analysis - void print_summary() const; - void print_detailed_results() const; - void print_statistics() const; - void print_brief_summary() const; - - // Get aggregate statistics - float get_total_avg_time_us() const; - float get_total_gflops() const; - size_t get_passed_count() const; - size_t get_failed_count() const; - - // Find best/worst performing results - const BenchmarkResult* get_fastest_result() const; - const BenchmarkResult* get_slowest_result() const; - const BenchmarkResult* get_highest_gflops_result() const; - - // Clear all results - void clear() { - results_.clear(); - } - - // Set operation name - void set_operation_name(const std::string& name) { - operation_name_ = name; - } - - private: - std::string operation_name_; - std::vector results_; - float gflops_; - bool correctness_passed_; -}; - -// -// Test case execution -// - -using FlopCalculatorFunc = std::function; - -// Default FLOP calculation function (assumes 1 FLOP per element) -int64_t default_flop_calculator(const TestCase& test_case); - -using ReferenceComputeFunc = std::function; - -// Half-open index range of the operator's own dispatch nodes within a -// benchmark graph's execute_nodes(). Staging upload nodes precede it, staging -// download nodes follow it. -struct OpNodeRange { - size_t begin = 0; - size_t end = 0; -}; - -// A benchmark graph plus the location of its repeatable operator nodes. The -// graph is heap-held: ComputeGraph owns its Context and must never be moved -// (a moved-from graph's destructor dereferences a null context). -struct BenchmarkGraph { - std::unique_ptr graph; - OpNodeRange op_nodes; -}; - -// Benchmark-only executor that records a graph's execute nodes into a single -// reusable command buffer, then replays it on every execute(). Staging uploads -// are encoded once, operator nodes N times, staging downloads once, so each -// iteration performs the same work as the old stacked-nodes layout (1 upload + -// N ops + 1 download) and the per-invocation divisor is unchanged. Production -// ComputeGraph::execute() behavior is unchanged. -// -// Notes for interpreting benchmark numbers: -// - Submit granularity differs from stacking N distinct nodes: all encodings -// live in one command buffer with one submit per iteration (the old path -// could split across command buffers at the node-count threshold), so -// per-dispatch times may shift systematically against older data. -// - Resize functions run once via propagate_resize() in execute_test_case -// before recording; replay itself never re-triggers resize. -// - Repeated encodings share one node/dispatch id, so per-repetition -// querypool attribution is unavailable (aggregation keys on kernel name). -class RepeatedGraphExecutor final { - public: - RepeatedGraphExecutor( - ComputeGraph& graph, - int repetitions, - OpNodeRange op_nodes); - void execute(); - - private: - ComputeGraph& graph_; - std::unique_ptr command_; -}; - -// Runs a measurement at the given chained_dispatches factor. The operator is -// built once, then its execute nodes are encoded that many times into a -// benchmark-only reusable command buffer. The probe-then-scale orchestration -// lives in execute_test_cases(). -// -// write_outputs controls whether the graph's staging output buffers are copied -// back into test_case.outputs() at the end of the run. The probe path needs -// write_outputs=true so the correctness check has a clean single-dispatch -// reference. The benchmarking path passes write_outputs=false to avoid the -// per-iter GPU->CPU copy cost. -BenchmarkResult execute_test_case( - TestCase& test_case, - int warmup_runs = 1, - int benchmark_runs = 1, - int chained_dispatches = 1, - bool write_outputs = true); - -TestResult execute_test_cases( - std::function()> test_case_generator, - FlopCalculatorFunc flop_calculator, - const std::string& operation_name = "Operation", - int warmup_runs = 1, - int benchmark_runs = 1, - ReferenceComputeFunc reference_compute_func = nullptr); - -TestResult execute_test_cases( - std::function()> test_case_generator, - const std::string& operation_name = "Operation", - int warmup_runs = 1, - int benchmark_runs = 1, - ReferenceComputeFunc reference_compute_func = nullptr); - -// -// Print utilities -// - -void print_performance_header(); -void print_separator(); - -void print_valuespec_data( - const ValueSpec& spec, - const std::string& name = "ValueSpec", - const bool print_ref_data = false, - size_t max_elements = 20, - int precision = 6); - -ValueRef quantized_weights_canvas( - ComputeGraph& graph, - const ValueRef weight_ref); - -ValueRef float_tensor_canvas(ComputeGraph& graph, const ValueRef weight_ref); - -// Compute weight sums for quantized operations (linear and convolution) -void compute_weight_sums( - ValueSpec& weight_sums, - const ValueSpec& quantized_weight, - int64_t out_features, - int64_t elements_per_output_feature); - -// Compute weight sums for 4D quantized conv2d operations -// Weight layout: [C_out, K_h, K_w, align_up_4(C_in_per_group)] -void compute_weight_sums_4d( - ValueSpec& weight_sums, - const ValueSpec& quantized_weight, - int64_t out_channels, - int64_t kernel_h, - int64_t kernel_w, - int64_t aligned_in_channels); - -// Compute weight sums for 4-bit group symmetric quantized weights -void compute_weight_sums_4bit_grouped( - ValueSpec& weight_sums, - const ValueSpec& quantized_weight, - int64_t num_groups, - int64_t out_features, - int64_t group_size); - -// Half-precision conversion utilities -uint16_t float_to_half(float value); -float half_to_float(uint16_t half_val); - -// Setup compute graph based on TestCase and operation name. The op function is -// invoked once. op_invocations_per_execute is used only to reserve enough -// descriptor capacity for benchmark-only repeated command encoding. Returns -// the graph plus the range of the operator's own nodes for repeated encoding. -BenchmarkGraph setup_compute_graph( - TestCase& test_case, - std::string op_name, - int op_invocations_per_execute = 1); - -} // namespace prototyping -} // namespace vulkan -} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/utils_test.cpp b/backends/vulkan/test/custom_ops/utils_test.cpp index 6d2de13d71e..104d585769e 100644 --- a/backends/vulkan/test/custom_ops/utils_test.cpp +++ b/backends/vulkan/test/custom_ops/utils_test.cpp @@ -6,7 +6,7 @@ #include -#include "utils.h" +#include namespace executorch::vulkan::prototyping { namespace { diff --git a/backends/vulkan/test/op_tests/conv2d_gemm_dynamic_test.cpp b/backends/vulkan/test/op_tests/conv2d_gemm_dynamic_test.cpp index e5a7216bfe7..7b4954113ea 100644 --- a/backends/vulkan/test/op_tests/conv2d_gemm_dynamic_test.cpp +++ b/backends/vulkan/test/op_tests/conv2d_gemm_dynamic_test.cpp @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include @@ -35,12 +35,13 @@ using namespace vkcompute; // // The im2col + GEMM conv path (conv2d_gemm_impl) materializes its im2col matrix // in tiles of output-height rows bounded to a byte budget. The static op-test -// suite (test/custom_ops/test_conv2d.cpp) only validates a single shape per -// graph. This test exercises the path under an actual trigger_resize: it builds -// the graph at an upper-bound input shape that forces MULTIPLE tiles, then -// resizes the input across tile boundaries (so trailing tiles must no-op via -// the shader's `oh < H_out` guard and the scratch must track the smaller shape) -// and verifies the output against a reference at every shape. +// suite (test/ops/benchmark/cases/conv2d/test_conv2d.cpp) only validates a +// single shape per graph. This test exercises the path under an actual +// trigger_resize: it builds the graph at an upper-bound input shape that forces +// MULTIPLE tiles, then resizes the input across tile boundaries (so trailing +// tiles must no-op via the shader's `oh < H_out` guard and the scratch must +// track the smaller shape) and verifies the output against a reference at every +// shape. // // Build-time upper bound: storage and the fixed num_tiles are built at the // initial input shape, so every resized shape MUST be <= the initial shape per diff --git a/backends/vulkan/test/ops/benchmark/BUCK b/backends/vulkan/test/ops/benchmark/BUCK new file mode 100644 index 00000000000..f559a6f1cfe --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/BUCK @@ -0,0 +1,9 @@ +# Any targets that should be shared between fbcode and xplat must be defined in +# targets.bzl. + +load("@fbsource//tools/build_defs:fbsource_utils.bzl", "is_fbcode") +load(":targets.bzl", "define_common_targets") + +oncall("executorch") + +define_common_targets(is_fbcode = is_fbcode()) diff --git a/backends/vulkan/test/custom_ops/CMakeLists.txt b/backends/vulkan/test/ops/benchmark/CMakeLists.txt similarity index 58% rename from backends/vulkan/test/custom_ops/CMakeLists.txt rename to backends/vulkan/test/ops/benchmark/CMakeLists.txt index f29b518ec06..522d4f95f46 100644 --- a/backends/vulkan/test/custom_ops/CMakeLists.txt +++ b/backends/vulkan/test/ops/benchmark/CMakeLists.txt @@ -25,7 +25,7 @@ message(STATUS "VULKAN_CXX_FLAGS: ${VULKAN_CXX_FLAGS}") # Only build if Vulkan was compiled if(TARGET vulkan_backend) if(NOT EXECUTORCH_ROOT) - set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../..) + set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../../..) endif() if(NOT PYTHON_EXECUTABLE) @@ -47,23 +47,32 @@ if(TARGET vulkan_backend) ) # Prototyping utility files - set(PROTOTYPING_UTILS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}) + set(PROTOTYPING_UTILS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/framework) set(PROTOTYPING_UTILS_CPP - ${CMAKE_CURRENT_SOURCE_DIR}/utils.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/conv2d_utils.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/cm_utils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/cm_utils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/config.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/conv2d_utils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/device_info.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/registry.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/results.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/runner.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/test_case.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/value_spec.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/framework/weight_utils.cpp ) # Prototyping shaders message(STATUS "shader stuff") - set(PROTOTYPING_SHADERS_PATH ${CMAKE_CURRENT_SOURCE_DIR}/glsl) + set(PROTOTYPING_SHADERS_PATH ${CMAKE_CURRENT_SOURCE_DIR}/test_ops/glsl) gen_vulkan_shader_lib_cpp(${PROTOTYPING_SHADERS_PATH}) vulkan_shader_lib(prototyping_shaderlib ${generated_spv_cpp}) target_compile_options(prototyping_shaderlib PRIVATE ${VULKAN_CXX_FLAGS}) message(STATUS "done shader stuff") # Operator implementations library - file(GLOB OPERATOR_IMPL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/impl/*.cpp) + file(GLOB OPERATOR_IMPL_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/test_ops/impl/*.cpp + ) add_library(operator_implementations STATIC ${OPERATOR_IMPL_SOURCES}) target_include_directories( operator_implementations PRIVATE ${COMMON_INCLUDES} @@ -78,34 +87,24 @@ if(TARGET vulkan_backend) executorch_target_link_options_shared_lib(vulkan_backend) executorch_target_link_options_shared_lib(operator_implementations) - # Function to create operator prototype binaries - function(add_operator_prototype OPERATOR_NAME) - set(TARGET_NAME ${OPERATOR_NAME}) - set(SOURCE_FILE ${CMAKE_CURRENT_SOURCE_DIR}/${OPERATOR_NAME}.cpp) - - add_executable(${TARGET_NAME} ${SOURCE_FILE} ${PROTOTYPING_UTILS_CPP}) - target_include_directories(${TARGET_NAME} PRIVATE ${COMMON_INCLUDES}) - target_link_libraries( - ${TARGET_NAME} PRIVATE vulkan_backend executorch_core - prototyping_shaderlib operator_implementations - ) - target_compile_options(${TARGET_NAME} PRIVATE ${VULKAN_CXX_FLAGS}) - set_property(TARGET ${TARGET_NAME} PROPERTY CXX_STANDARD 17) - endfunction() - - # Define operator prototypes - add_operator_prototype(test_add) - add_operator_prototype(test_q8csw_linear) - add_operator_prototype(test_q8csw_conv2d) - add_operator_prototype(test_q4gsw_linear) - add_operator_prototype(test_choose_qparams_per_row) - add_operator_prototype(test_q8ta_qdq) - add_operator_prototype(test_q8ta_clone) - add_operator_prototype(test_q8ta_binary) - add_operator_prototype(test_q8ta_conv2d) - add_operator_prototype(test_q8ta_conv2d_pw) - add_operator_prototype(test_q8ta_conv2d_dw) - add_operator_prototype(test_mm) - add_operator_prototype(test_coopmat_probe) - add_operator_prototype(test_coopmat_linear_bench) + # gflags is built by the main ExecuTorch build but not installed, so find it + # in that build tree, which build_and_run.sh places five levels above this + # project's build directory. + set(gflags_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../../../third-party/gflags) + find_package(gflags REQUIRED) + + file(GLOB_RECURSE ETVK_BENCH_CASES_CPP + ${CMAKE_CURRENT_SOURCE_DIR}/cases/*.cpp + ) + add_executable( + etvk_bench ${CMAKE_CURRENT_SOURCE_DIR}/etvk_bench.cpp + ${ETVK_BENCH_CASES_CPP} ${PROTOTYPING_UTILS_CPP} + ) + target_include_directories(etvk_bench PRIVATE ${COMMON_INCLUDES}) + target_link_libraries( + etvk_bench PRIVATE vulkan_backend executorch_core prototyping_shaderlib + operator_implementations gflags + ) + target_compile_options(etvk_bench PRIVATE ${VULKAN_CXX_FLAGS}) + set_property(TARGET etvk_bench PROPERTY CXX_STANDARD 17) endif() diff --git a/backends/vulkan/test/custom_ops/build_and_run.sh b/backends/vulkan/test/ops/benchmark/build_and_run.sh similarity index 89% rename from backends/vulkan/test/custom_ops/build_and_run.sh rename to backends/vulkan/test/ops/benchmark/build_and_run.sh index 2641970faf8..a03aa3032a3 100755 --- a/backends/vulkan/test/custom_ops/build_and_run.sh +++ b/backends/vulkan/test/ops/benchmark/build_and_run.sh @@ -49,28 +49,28 @@ configure_and_build_tests() { local android_args="" if [[ "$ANDROID_MODE" == "true" ]]; then - cmake backends/vulkan/test/custom_ops/ \ + cmake backends/vulkan/test/ops/benchmark/ \ -DCMAKE_INSTALL_PREFIX=$CMAKE_OUT_DIR \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \ -DANDROID_ABI=arm64-v8a \ -DANDROID_PLATFORM=android-28 \ -DGLSLC_PATH=$(which glslc) \ - -B$CMAKE_OUT_DIR/backends/vulkan/test/custom_ops + -B$CMAKE_OUT_DIR/backends/vulkan/test/ops/benchmark else - cmake backends/vulkan/test/custom_ops/ \ + cmake backends/vulkan/test/ops/benchmark/ \ -DCMAKE_INSTALL_PREFIX=$CMAKE_OUT_DIR \ -DCMAKE_BUILD_TYPE=Debug \ -DGLSLC_PATH=$(which glslc) \ - -B$CMAKE_OUT_DIR/backends/vulkan/test/custom_ops + -B$CMAKE_OUT_DIR/backends/vulkan/test/ops/benchmark fi - cmake --build $CMAKE_OUT_DIR/backends/vulkan/test/custom_ops -j$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu) + 1 )) --target all + cmake --build $CMAKE_OUT_DIR/backends/vulkan/test/ops/benchmark -j$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu) + 1 )) --target all } build_tests() { - cmake --build $CMAKE_OUT_DIR/backends/vulkan/test/custom_ops -j$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu) + 1 )) --target all + cmake --build $CMAKE_OUT_DIR/backends/vulkan/test/ops/benchmark -j$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu) + 1 )) --target all } # Function to rebuild both main and tests @@ -90,7 +90,7 @@ clean_and_rebuild() { execute_binary() { local binary_name="$1" if [[ -n "$binary_name" ]]; then - local binary_path="$CMAKE_OUT_DIR/backends/vulkan/test/custom_ops/$binary_name" + local binary_path="$CMAKE_OUT_DIR/backends/vulkan/test/ops/benchmark/$binary_name" echo "Executing binary: $binary_path" if [[ "$ANDROID_MODE" == "true" ]]; then diff --git a/backends/vulkan/test/ops/benchmark/cases/add/add.cpp b/backends/vulkan/test/ops/benchmark/cases/add/add.cpp new file mode 100644 index 00000000000..1f54501ae05 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/add/add.cpp @@ -0,0 +1,60 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace add { + +// Custom FLOP calculator for add operation +// Add operation performs 1 FLOP (addition) per element +int64_t add_flop_calculator(const TestCase& test_case) { + // Calculate total elements from the first input tensor + int64_t total_elements = 1; + if (!test_case.empty() && test_case.num_inputs() > 0 && + test_case.inputs()[0].is_tensor()) { + const auto& sizes = test_case.inputs()[0].get_tensor_sizes(); + for (int64_t size : sizes) { + total_elements *= size; + } + } + + // Add operation: 1 FLOP per element (one addition) + return total_elements; +} + +// Reference implementation for add operator +void add_reference_compute(TestCase& test_case) { + const ValueSpec& input_a = test_case.inputs().at(0); + const ValueSpec& input_b = test_case.inputs().at(1); + + ValueSpec& output = test_case.outputs().at(0); + + if (input_a.dtype != vkapi::kFloat) { + throw std::invalid_argument("Unsupported dtype"); + } + + // Calculate number of elements + int64_t num_elements = input_a.numel(); + + auto& input_a_data = input_a.get_float_data(); + auto& input_b_data = input_b.get_float_data(); + + auto& ref_data = output.get_ref_float_data(); + ref_data.resize(num_elements); + for (int64_t i = 0; i < num_elements; ++i) { + ref_data[i] = input_a_data[i] + input_b_data[i]; + } +} + +} // namespace add +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/add/add.h b/backends/vulkan/test/ops/benchmark/cases/add/add.h new file mode 100644 index 00000000000..ae311090992 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/add/add.h @@ -0,0 +1,24 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace add { + +int64_t add_flop_calculator(const TestCase& test_case); + +void add_reference_compute(TestCase& test_case); + +} // namespace add +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/add/test_sets/correctness_set.cpp b/backends/vulkan/test/ops/benchmark/cases/add/test_sets/correctness_set.cpp new file mode 100644 index 00000000000..78da70f50b2 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/add/test_sets/correctness_set.cpp @@ -0,0 +1,106 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace add { + +namespace { + +std::vector generate_test_cases( + const std::vector>& size_configs, + const std::vector& storage_types, + const std::vector& data_types) { + std::vector test_cases; + + // Set the data generation type as a local variable + DataGenType data_gen_type = DataGenType::ONES; + + // Generate test cases for each combination + for (const auto& sizes : size_configs) { + for (const auto& storage_type : storage_types) { + for (const auto& data_type : data_types) { + TestCase test_case; + + // Create a descriptive name for the test case + std::string shape_str = + shape_bracket(sizes) + "+" + shape_bracket(sizes); + std::string storage_str = repr_str(storage_type, utils::kWidthPacked); + std::string dtype_str = dtype_short(data_type); + std::string test_name = make_test_label( + "ACCU", dtype_str, dtype_str, shape_str, storage_str); + test_case.set_name(test_name); + + // Set the operator name for the test case + test_case.set_operator_name("etvk.add_prototype"); + + // Add two input tensors with the same size, type, storage, and data + // generation method + ValueSpec input_a( + sizes, data_type, storage_type, utils::kWidthPacked, data_gen_type); + ValueSpec input_b( + sizes, data_type, storage_type, utils::kWidthPacked, data_gen_type); + + // Add output tensor with the same size, type, and storage as inputs + // (output uses ZEROS by default) + ValueSpec output( + sizes, + data_type, + storage_type, + utils::kWidthPacked, + DataGenType::ZEROS); + + test_case.add_input_spec(input_a); + test_case.add_input_spec(input_b); + test_case.add_output_spec(output); + + test_cases.push_back(test_case); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("add", "correctness") { + // Define different input size configurations + const std::vector> size_configs = { + {1, 64, 64}, // Small square + {1, 128, 128}, // Medium square + {1, 256, 256}, // Large square + {1, 512, 512}, // Very large square + {1, 1, 1024}, // Wide tensor + {1, 1024, 1}, // Tall tensor + {32, 32, 32}, // 3D cube + {16, 128, 64}, // 3D rectangular + }; + + // Storage types to test + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + + // Data types to test + const std::vector data_types = { + vkapi::kFloat, vkapi::kHalf}; + + return { + generate_test_cases(size_configs, storage_types, data_types), + add_reference_compute, + add_flop_calculator}; +} + +} // namespace add +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_choose_qparams_per_row.cpp b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/choose_qparams_per_row.cpp similarity index 71% rename from backends/vulkan/test/custom_ops/test_choose_qparams_per_row.cpp rename to backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/choose_qparams_per_row.cpp index 69f7812e4ab..fa96168c04a 100644 --- a/backends/vulkan/test/custom_ops/test_choose_qparams_per_row.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/choose_qparams_per_row.cpp @@ -4,31 +4,20 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include +#include + #include #include -#include +#include +#include +#include +#include #include -#include "utils.h" - -#include - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 2050; -static constexpr float SMALL_SCALE_THRESHOLD = 6.1e-5f; -// ChooseQParams configuration struct -struct ChooseQParamsConfig { - int64_t num_channels; // Height dimension (number of channels) - int64_t channel_size; // Width dimension (size per channel) - int32_t quant_min = -128; - int32_t quant_max = 127; - std::string test_case_name = "placeholder"; - std::string op_name = "choose_qparams_per_row"; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace choose_qparams_per_row { // Utility function to create a test case from a ChooseQParamsConfig TestCase create_test_case_from_config( @@ -99,6 +88,26 @@ TestCase create_test_case_from_config( return test_case; } +int64_t choose_qparams_per_channel_flop_calculator(const TestCase& test_case) { + // Get input dimensions + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + int64_t num_channels = input_sizes[0]; + int64_t channel_size = input_sizes[1]; + + // Calculate FLOPs for choose_qparams_per_channel operation + // Each channel requires: + // - Min/max finding: approximately 2 * channel_size comparisons + // - Scale calculation: ~5 operations (division, min/max operations) + // - Zero point calculation: ~10 operations (multiple arithmetic operations) + int64_t ops_per_channel = 2 * channel_size + 15; // Simplified estimate + + int64_t flop = num_channels * ops_per_channel; + + return flop; +} + +static constexpr float SMALL_SCALE_THRESHOLD = 6.1e-5f; + // CPU reference implementation matching the behavior from op_choose_qparams.cpp void calculate_scale_and_zero_point_reference( float min_val, @@ -171,90 +180,14 @@ void calculate_scale_and_zero_point_reference( } else if (initial_zero_point > qmax) { nudged_zero_point = qmax; } else { - nudged_zero_point = - static_cast(nearbyint(static_cast(initial_zero_point))); + nudged_zero_point = static_cast( + std::nearbyint(static_cast(initial_zero_point))); } scale = static_cast(scale_double); zero_point = nudged_zero_point; } -// Generate easy test cases for choose_qparams_per_channel operation (for -// debugging) -std::vector generate_choose_qparams_per_channel_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - int num_channels = 4; - int channel_size = 8; - - ChooseQParamsConfig config = { - num_channels, // num_channels - channel_size, // channel_size - -128, // quant_min - 127, // quant_max - "simple", // test_case_name - }; - - // Test with both storage types - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - std::vector float_types = {vkapi::kFloat}; - - // Generate test cases for each combination - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, input_dtype)); - } - } - - return test_cases; -} - -// Generate test cases for choose_qparams_per_channel operation -std::vector generate_choose_qparams_per_channel_test_cases() { - std::vector test_cases; - - std::vector configs = { - {4, 16}, - {8, 32}, - {16, 64}, - {32, 128}, - {64, 256}, - {128, 512}, - {1, 512}, - // Performance cases - {256, 1024}, - {512, 2048}, - {1, 2048}, - {1, 8096}, - }; - - // Test with different storage types - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - for (auto config : configs) { - std::string prefix = (config.num_channels < kRefDimSizeLimit && - config.channel_size < kRefDimSizeLimit) - ? "correctness_" - : "performance_"; - std::string generated_test_case_name = prefix + - std::to_string(config.num_channels) + "_" + - std::to_string(config.channel_size); - - config.test_case_name = generated_test_case_name; - - for (const auto& storage_type : storage_types) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, vkapi::kFloat)); - } - } - - return test_cases; -} - // Reference implementation for choose_qparams_per_channel void choose_qparams_per_channel_reference_impl(TestCase& test_case) { int32_t idx = 0; @@ -322,44 +255,7 @@ void reference_impl(TestCase& test_case) { choose_qparams_per_channel_reference_impl(test_case); } -int64_t choose_qparams_per_channel_flop_calculator(const TestCase& test_case) { - // Get input dimensions - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - int64_t num_channels = input_sizes[0]; - int64_t channel_size = input_sizes[1]; - - // Calculate FLOPs for choose_qparams_per_channel operation - // Each channel requires: - // - Min/max finding: approximately 2 * channel_size comparisons - // - Scale calculation: ~5 operations (division, min/max operations) - // - Zero point calculation: ~10 operations (multiple arithmetic operations) - int64_t ops_per_channel = 2 * channel_size + 15; // Simplified estimate - - int64_t flop = num_channels * ops_per_channel; - - return flop; -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Choose QParams Per Channel Operation Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_choose_qparams_per_channel_test_cases, - choose_qparams_per_channel_flop_calculator, - "ChooseQParamsPerChannel", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace choose_qparams_per_row +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/choose_qparams_per_row.h b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/choose_qparams_per_row.h new file mode 100644 index 00000000000..88ce7f05028 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/choose_qparams_per_row.h @@ -0,0 +1,43 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace choose_qparams_per_row { + +constexpr int64_t kRefDimSizeLimit = 2050; + +// ChooseQParams configuration struct +struct ChooseQParamsConfig { + int64_t num_channels; // Height dimension (number of channels) + int64_t channel_size; // Width dimension (size per channel) + int32_t quant_min = -128; + int32_t quant_max = 127; + std::string test_case_name = "placeholder"; + std::string op_name = "choose_qparams_per_row"; +}; + +TestCase create_test_case_from_config( + const ChooseQParamsConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype); + +void reference_impl(TestCase& test_case); + +int64_t choose_qparams_per_channel_flop_calculator(const TestCase& test_case); + +} // namespace choose_qparams_per_row +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/test_sets/choose_qparams_per_row_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/test_sets/choose_qparams_per_row_sets.cpp new file mode 100644 index 00000000000..6b95e4ff303 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/choose_qparams_per_row/test_sets/choose_qparams_per_row_sets.cpp @@ -0,0 +1,99 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace choose_qparams_per_row { + +namespace { + +std::vector generate_test_cases( + const std::vector& configs, + const std::vector& storage_types) { + std::vector test_cases; + + for (auto config : configs) { + std::string prefix = (config.num_channels < kRefDimSizeLimit && + config.channel_size < kRefDimSizeLimit) + ? "correctness_" + : "performance_"; + std::string generated_test_case_name = prefix + + std::to_string(config.num_channels) + "_" + + std::to_string(config.channel_size); + + config.test_case_name = generated_test_case_name; + + for (const auto& storage_type : storage_types) { + test_cases.push_back( + create_test_case_from_config(config, storage_type, vkapi::kFloat)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("choose_qparams_per_row", "correctness") { + const std::vector configs = { + {4, 16}, + {8, 32}, + {16, 64}, + {32, 128}, + {64, 256}, + {128, 512}, + {1, 512}, + {256, 1024}, + {512, 2048}, + {1, 2048}, + }; + // Test with different storage types + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + return { + generate_test_cases(configs, storage_types), + reference_impl, + choose_qparams_per_channel_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("choose_qparams_per_row", "debug") { + const std::vector configs = { + // Single simple configuration for debugging + {4, 8}, + }; + // Test with different storage types + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + return { + generate_test_cases(configs, storage_types), + reference_impl, + choose_qparams_per_channel_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("choose_qparams_per_row", "performance") { + const std::vector configs = { + // Performance cases + {1, 8096}, + }; + // Test with different storage types + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + return { + generate_test_cases(configs, storage_types), + reference_impl, + choose_qparams_per_channel_flop_calculator}; +} + +} // namespace choose_qparams_per_row +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_conv1d_dw.cpp b/backends/vulkan/test/ops/benchmark/cases/conv1d/conv1d.cpp similarity index 52% rename from backends/vulkan/test/custom_ops/test_conv1d_dw.cpp rename to backends/vulkan/test/ops/benchmark/cases/conv1d/conv1d.cpp index 92539037f75..136f9fa6cf2 100644 --- a/backends/vulkan/test/custom_ops/test_conv1d_dw.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/conv1d/conv1d.cpp @@ -4,31 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include - -#include -#include - -#include "utils.h" +#include -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 256; +#include +#include +#include -struct Conv1dDWConfig { - int64_t N; - int64_t C; - int64_t L; - int64_t K; - int64_t stride; - int64_t padding; - int64_t dilation; - bool has_bias; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv1d { -static TestCase create_conv1d_dw_test_case( +TestCase create_conv1d_dw_test_case( const Conv1dDWConfig& config, vkapi::ScalarType dtype, utils::StorageType storage_type) { @@ -126,7 +113,123 @@ static TestCase create_conv1d_dw_test_case( return test_case; } -static void conv1d_dw_reference_impl(TestCase& test_case) { +TestCase create_conv1d_pw_test_case( + const Conv1dPWConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type) { + TestCase test_case; + + bool is_perf = config.C_in > kRefDimSizeLimit || + config.C_out > kRefDimSizeLimit || config.L > kRefDimSizeLimit; + + std::string prefix = is_perf ? "PERF" : "ACCU"; + std::string storage_str = storage_type_abbrev(storage_type) + "(HP)"; + std::string dtype_str = dtype_short(dtype); + + std::string bias_str = config.has_bias ? "+bias" : ""; + + std::string shape = "[" + std::to_string(config.N) + "," + + std::to_string(config.C_in) + "," + std::to_string(config.L) + "]x[" + + std::to_string(config.C_out) + "," + std::to_string(config.C_in) + ",1]"; + + std::string name = make_test_label( + prefix, dtype_str, dtype_str, shape, storage_str, bias_str); + + test_case.set_name(name); + test_case.set_operator_name("test_etvk.test_conv1d_pw.default"); + + // Input: [N, C_in, L] height-packed + ValueSpec input( + {config.N, config.C_in, config.L}, + dtype, + storage_type, + utils::kHeightPacked, + DataGenType::RANDOM); + test_case.add_input_spec(input); + + // Weight: [C_out, C_in, 1] height-packed, constant + ValueSpec weight( + {config.C_out, config.C_in, 1}, + dtype, + storage_type, + utils::kHeightPacked, + DataGenType::RANDOM); + weight.set_constant(true); + test_case.add_input_spec(weight); + + // Bias: [C_out] or None + if (config.has_bias) { + ValueSpec bias( + {config.C_out}, + dtype, + storage_type, + utils::kWidthPacked, + DataGenType::RANDOM); + bias.set_constant(true); + test_case.add_input_spec(bias); + } else { + ValueSpec none_bias(static_cast(0)); + none_bias.set_none(true); + test_case.add_input_spec(none_bias); + } + + // stride = [1] + test_case.add_input_spec(ValueSpec(std::vector{1})); + // padding = [0] + test_case.add_input_spec(ValueSpec(std::vector{0})); + // dilation = [1] + test_case.add_input_spec(ValueSpec(std::vector{1})); + // groups = 1 + test_case.add_input_spec(ValueSpec(static_cast(1))); + + // Output: [N, C_out, L] height-packed + ValueSpec output( + {config.N, config.C_out, config.L}, + dtype, + storage_type, + utils::kHeightPacked, + DataGenType::ZEROS); + test_case.add_output_spec(output); + + if (dtype == vkapi::kHalf) { + test_case.set_abs_tolerance(1e-1f); + test_case.set_rel_tolerance(1e-2f); + } else { + test_case.set_abs_tolerance(1e-3f); + test_case.set_rel_tolerance(1e-3f); + } + + test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); + + return test_case; +} + +int64_t conv1d_dw_flop_calculator(const TestCase& test_case) { + auto in_sizes = test_case.inputs()[0].get_tensor_sizes(); + auto w_sizes = test_case.inputs()[1].get_tensor_sizes(); + auto out_sizes = test_case.outputs()[0].get_tensor_sizes(); + + const int64_t N = in_sizes[0]; + const int64_t C = in_sizes[1]; + const int64_t K = w_sizes[2]; + const int64_t L_out = out_sizes[2]; + + return 2 * N * C * L_out * K; +} + +int64_t conv1d_pw_flop_calculator(const TestCase& test_case) { + auto in_sizes = test_case.inputs()[0].get_tensor_sizes(); + auto w_sizes = test_case.inputs()[1].get_tensor_sizes(); + + int64_t N = in_sizes[0]; + int64_t C_in = in_sizes[1]; + int64_t L = in_sizes[2]; + int64_t C_out = w_sizes[0]; + + return 2 * N * C_in * C_out * L; +} + +void conv1d_dw_reference_impl(TestCase& test_case) { const auto& input_spec = test_case.inputs()[0]; const auto& weight_spec = test_case.inputs()[1]; const auto& bias_spec = test_case.inputs()[2]; @@ -185,83 +288,56 @@ static void conv1d_dw_reference_impl(TestCase& test_case) { } } -static std::vector generate_conv1d_dw_test_cases() { - std::vector test_cases; - - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - // Accuracy shapes - std::vector accu_configs = { - // {N, C, L, K, stride, padding, dilation, has_bias} - {1, 16, 64, 3, 1, 1, 1, false}, - {1, 32, 128, 5, 1, 2, 1, true}, - {1, 64, 32, 3, 2, 1, 1, false}, - {2, 16, 64, 3, 1, 1, 1, true}, - {1, 16, 64, 7, 1, 3, 2, false}, - // Non-aligned channel counts (not a multiple of 4) - {1, 5, 64, 3, 1, 1, 1, false}, - {1, 5, 64, 3, 1, 1, 1, true}, - {1, 7, 32, 5, 1, 2, 1, false}, - {1, 13, 48, 3, 2, 1, 1, true}, - {2, 7, 64, 3, 1, 1, 1, false}, - }; - - for (const auto& cfg : accu_configs) { - for (auto st : storage_types) { - test_cases.push_back(create_conv1d_dw_test_case(cfg, vkapi::kFloat, st)); - } - } +void conv1d_pw_reference_impl(TestCase& test_case) { + const auto& input_spec = test_case.inputs()[0]; + const auto& weight_spec = test_case.inputs()[1]; + const auto& bias_spec = test_case.inputs()[2]; + ValueSpec& output = test_case.outputs()[0]; - // Performance shapes (half + float) - std::vector perf_configs = { - {1, 256, 1024, 3, 1, 1, 1, false}, - {1, 512, 2048, 5, 1, 2, 1, true}, - {1, 128, 4096, 31, 1, 15, 1, false}, - }; - - for (const auto& cfg : perf_configs) { - for (auto st : storage_types) { - test_cases.push_back(create_conv1d_dw_test_case(cfg, vkapi::kFloat, st)); - test_cases.push_back(create_conv1d_dw_test_case(cfg, vkapi::kHalf, st)); - } + if (input_spec.dtype != vkapi::kFloat) { + throw std::invalid_argument("Reference only supports float"); } - return test_cases; -} - -static int64_t conv1d_dw_flop_calculator(const TestCase& test_case) { - auto in_sizes = test_case.inputs()[0].get_tensor_sizes(); - auto w_sizes = test_case.inputs()[1].get_tensor_sizes(); - auto out_sizes = test_case.outputs()[0].get_tensor_sizes(); - - const int64_t N = in_sizes[0]; - const int64_t C = in_sizes[1]; - const int64_t K = w_sizes[2]; - const int64_t L_out = out_sizes[2]; - - return 2 * N * C * L_out * K; -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); + auto in_sizes = input_spec.get_tensor_sizes(); + auto w_sizes = weight_spec.get_tensor_sizes(); - print_performance_header(); - std::cout << "Conv1d Depthwise (Height-Packed) Benchmark" << std::endl; - print_separator(); + int64_t N = in_sizes[0]; + int64_t C_in = in_sizes[1]; + int64_t L = in_sizes[2]; + int64_t C_out = w_sizes[0]; - ReferenceComputeFunc ref_fn = conv1d_dw_reference_impl; + const auto& in_data = input_spec.get_float_data(); + const auto& w_data = weight_spec.get_float_data(); + auto& ref_data = output.get_ref_float_data(); + ref_data.resize(N * C_out * L, 0.0f); - auto results = execute_test_cases( - generate_conv1d_dw_test_cases, - conv1d_dw_flop_calculator, - "Conv1dDW", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); + // input is NCHW-contiguous: [N, C_in, L] + // weight is [C_out, C_in, 1] + for (int64_t n = 0; n < N; ++n) { + for (int64_t oc = 0; oc < C_out; ++oc) { + for (int64_t l = 0; l < L; ++l) { + float sum = 0.0f; + for (int64_t ic = 0; ic < C_in; ++ic) { + sum += in_data[n * C_in * L + ic * L + l] * w_data[oc * C_in + ic]; + } + ref_data[n * C_out * L + oc * L + l] = sum; + } + } + } - return 0; + if (!bias_spec.is_none()) { + const auto& bias_data = bias_spec.get_float_data(); + for (int64_t n = 0; n < N; ++n) { + for (int64_t oc = 0; oc < C_out; ++oc) { + for (int64_t l = 0; l < L; ++l) { + ref_data[n * C_out * L + oc * L + l] += bias_data[oc]; + } + } + } + } } + +} // namespace conv1d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv1d/conv1d.h b/backends/vulkan/test/ops/benchmark/cases/conv1d/conv1d.h new file mode 100644 index 00000000000..c915e9c91d9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv1d/conv1d.h @@ -0,0 +1,59 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv1d { + +constexpr int64_t kRefDimSizeLimit = 256; + +struct Conv1dDWConfig { + int64_t N; + int64_t C; + int64_t L; + int64_t K; + int64_t stride; + int64_t padding; + int64_t dilation; + bool has_bias; +}; + +struct Conv1dPWConfig { + int64_t N; + int64_t C_in; + int64_t C_out; + int64_t L; + bool has_bias; +}; + +TestCase create_conv1d_dw_test_case( + const Conv1dDWConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type); + +TestCase create_conv1d_pw_test_case( + const Conv1dPWConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type); + +void conv1d_dw_reference_impl(TestCase& test_case); + +void conv1d_pw_reference_impl(TestCase& test_case); + +int64_t conv1d_dw_flop_calculator(const TestCase& test_case); + +int64_t conv1d_pw_flop_calculator(const TestCase& test_case); + +} // namespace conv1d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/dw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/dw_sets.cpp new file mode 100644 index 00000000000..24ed642e6aa --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/dw_sets.cpp @@ -0,0 +1,81 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv1d { + +namespace { + +std::vector generate_dw_test_cases( + const std::vector& configs, + const std::vector& storage_types, + const std::vector& dtypes) { + std::vector test_cases; + + for (const auto& cfg : configs) { + for (auto st : storage_types) { + for (auto dtype : dtypes) { + test_cases.push_back(create_conv1d_dw_test_case(cfg, dtype, st)); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("conv1d", "dw_correctness") { + // Accuracy shapes + const std::vector configs = { + // {N, C, L, K, stride, padding, dilation, has_bias} + {1, 16, 64, 3, 1, 1, 1, false}, + {1, 32, 128, 5, 1, 2, 1, true}, + {1, 64, 32, 3, 2, 1, 1, false}, + {2, 16, 64, 3, 1, 1, 1, true}, + {1, 16, 64, 7, 1, 3, 2, false}, + // Non-aligned channel counts (not a multiple of 4) + {1, 5, 64, 3, 1, 1, 1, false}, + {1, 5, 64, 3, 1, 1, 1, true}, + {1, 7, 32, 5, 1, 2, 1, false}, + {1, 13, 48, 3, 2, 1, 1, true}, + {2, 7, 64, 3, 1, 1, 1, false}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector dtypes = {vkapi::kFloat}; + return { + generate_dw_test_cases(configs, storage_types, dtypes), + conv1d_dw_reference_impl, + conv1d_dw_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("conv1d", "dw_performance") { + // Performance shapes (half + float) + const std::vector configs = { + {1, 256, 1024, 3, 1, 1, 1, false}, + {1, 512, 2048, 5, 1, 2, 1, true}, + {1, 128, 4096, 31, 1, 15, 1, false}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; + return { + generate_dw_test_cases(configs, storage_types, dtypes), + conv1d_dw_reference_impl, + conv1d_dw_flop_calculator}; +} + +} // namespace conv1d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/pw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/pw_sets.cpp new file mode 100644 index 00000000000..67f6b873425 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv1d/test_sets/pw_sets.cpp @@ -0,0 +1,85 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv1d { + +namespace { + +std::vector generate_pw_test_cases( + const std::vector& configs, + const std::vector& storage_types, + const std::vector& dtypes) { + std::vector test_cases; + + for (const auto& cfg : configs) { + for (auto st : storage_types) { + for (auto dtype : dtypes) { + test_cases.push_back(create_conv1d_pw_test_case(cfg, dtype, st)); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("conv1d", "pw_correctness") { + // Accuracy shapes (float, small) + const std::vector configs = { + {1, 16, 32, 64, false}, + {1, 16, 32, 64, true}, + {1, 32, 16, 128, false}, + {1, 32, 16, 128, true}, + {1, 64, 64, 32, false}, + {1, 128, 256, 16, true}, + {2, 16, 32, 64, false}, + {2, 16, 32, 64, true}, + // Non-aligned channel counts (not a multiple of 4) + {1, 5, 7, 64, false}, + {1, 5, 7, 64, true}, + {1, 13, 17, 48, false}, + {1, 13, 17, 48, true}, + {1, 7, 5, 32, false}, + {2, 5, 13, 64, true}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector dtypes = {vkapi::kFloat}; + return { + generate_pw_test_cases(configs, storage_types, dtypes), + conv1d_pw_reference_impl, + conv1d_pw_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("conv1d", "pw_performance") { + // Performance shapes (half + float) + const std::vector configs = { + {1, 256, 512, 1024, false}, + {1, 256, 512, 1024, true}, + {1, 512, 256, 2048, false}, + {1, 128, 128, 4096, true}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; + return { + generate_pw_test_cases(configs, storage_types, dtypes), + conv1d_pw_reference_impl, + conv1d_pw_flop_calculator}; +} + +} // namespace conv1d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.cpp new file mode 100644 index 00000000000..8421257c40b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.cpp @@ -0,0 +1,639 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv2d { + +namespace { + +int64_t calc_out_size( + int64_t in_size, + int64_t kernel_size, + int64_t stride, + int64_t padding, + int64_t dilation) { + return (in_size + 2 * padding - dilation * (kernel_size - 1) - 1) / stride + + 1; +} + +} // namespace + +// Shared perf/skip classification used by both create_conv2d_test_case (to tag +// PERF vs ACCU) and conv2d_reference_impl (to gate the large-K FP16 reference +// check). A shape is "perf" if any dimension reaches kRefDimSizeLimit; the +// boundary is inclusive (>=) so a 64-wide dim counts as perf — FP16 +// accumulation error at K = K_h * K_w * C_in for such shapes can exceed the +// half tolerance and false-fail. Keep both call sites on this single helper to +// avoid the two predicates drifting apart. +bool conv2d_is_perf_shape(int64_t C_in, int64_t C_out, int64_t H, int64_t W) { + return C_in >= kRefDimSizeLimit || C_out >= kRefDimSizeLimit || + H >= kRefDimSizeLimit || W >= kRefDimSizeLimit; +} + +TestCase create_conv2d_test_case( + const Conv2dTestConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout, + const std::string& impl_selector) { + TestCase test_case; + + bool is_perf = conv2d_is_perf_shape( + config.dims.C, config.C_out, config.dims.H, config.dims.W); + + std::string prefix = is_perf ? "PERF" : "ACCU"; + std::string storage_str = repr_str(storage_type, memory_layout); + std::string dtype_str = dtype_short(dtype); + std::string bias_str = config.has_bias ? "+bias" : ""; + + int64_t H_out = calc_out_size( + config.dims.H, + config.kernel.h, + config.stride.h, + config.padding.h, + config.dilation.h); + int64_t W_out = calc_out_size( + config.dims.W, + config.kernel.w, + config.stride.w, + config.padding.w, + config.dilation.w); + + std::string shape = "[" + std::to_string(config.dims.N) + "," + + std::to_string(config.dims.C) + "," + std::to_string(config.dims.H) + + "," + std::to_string(config.dims.W) + "]->[" + + std::to_string(config.C_out) + "] k" + std::to_string(config.kernel.h) + + "x" + std::to_string(config.kernel.w) + " s" + + std::to_string(config.stride.h) + " p" + + std::to_string(config.padding.h) + " d" + + std::to_string(config.dilation.h); + + std::string suffix = bias_str; + if (!impl_selector.empty()) { + if (!suffix.empty()) { + suffix += " "; + } + suffix += "[" + impl_selector + "]"; + } + + std::string name = + make_test_label(prefix, dtype_str, dtype_str, shape, storage_str, suffix); + + test_case.set_name(name); + test_case.set_operator_name("test_etvk.test_conv2d.default"); + + // Input tensor [N, C_in, H, W] + ValueSpec input( + {config.dims.N, config.dims.C, config.dims.H, config.dims.W}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + + // Weight tensor [C_out, C_in, K_h, K_w] - constant + ValueSpec weight( + {config.C_out, config.dims.C, config.kernel.h, config.kernel.w}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + weight.set_constant(true); + + test_case.add_input_spec(input); + test_case.add_input_spec(weight); + + // Bias (or none) + if (config.has_bias) { + ValueSpec bias( + {config.C_out}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + bias.set_constant(true); + test_case.add_input_spec(bias); + } else { + ValueSpec none_bias(static_cast(0)); + none_bias.set_none(true); + test_case.add_input_spec(none_bias); + } + + // stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w + test_case.add_input_spec(ValueSpec(static_cast(config.stride.h))); + test_case.add_input_spec(ValueSpec(static_cast(config.stride.w))); + test_case.add_input_spec(ValueSpec(static_cast(config.padding.h))); + test_case.add_input_spec(ValueSpec(static_cast(config.padding.w))); + test_case.add_input_spec(ValueSpec(static_cast(config.dilation.h))); + test_case.add_input_spec(ValueSpec(static_cast(config.dilation.w))); + + // impl_selector string + test_case.add_input_spec(ValueSpec::make_string(impl_selector)); + + // Output tensor [N, C_out, H_out, W_out] + ValueSpec output( + {config.dims.N, config.C_out, H_out, W_out}, + dtype, + storage_type, + memory_layout, + DataGenType::ZEROS); + test_case.add_output_spec(output); + + if (dtype == vkapi::kHalf) { + test_case.set_abs_tolerance(1e-1f); + test_case.set_rel_tolerance(1e-2f); + } else { + test_case.set_abs_tolerance(1e-3f); + test_case.set_rel_tolerance(1e-3f); + } + + test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); + + return test_case; +} + +TestCase create_conv2d_dw_test_case( + const Conv2dDwConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout, + const std::string& impl_selector) { + TestCase test_case; + + bool is_perf = config.dims.C > kRefDimSizeLimit || + config.dims.H > kRefDimSizeLimit || config.dims.W > kRefDimSizeLimit; + + std::string prefix = is_perf ? "PERF" : "ACCU"; + std::string storage_str = repr_str(storage_type, memory_layout); + std::string dtype_str = dtype_short(dtype); + std::string bias_str = config.has_bias ? "+bias" : ""; + + int64_t H_out = calc_out_size( + config.dims.H, + config.kernel.h, + config.stride.h, + config.padding.h, + config.dilation.h); + int64_t W_out = calc_out_size( + config.dims.W, + config.kernel.w, + config.stride.w, + config.padding.w, + config.dilation.w); + (void)H_out; + (void)W_out; + + // groups for depthwise conv2d == number of input channels + std::string shape = "[" + std::to_string(config.dims.N) + "," + + std::to_string(config.dims.C) + "," + std::to_string(config.dims.H) + + "," + std::to_string(config.dims.W) + "] k" + + std::to_string(config.kernel.h) + " s" + std::to_string(config.stride.h) + + " p" + std::to_string(config.padding.h) + " d" + + std::to_string(config.dilation.h) + " g" + std::to_string(config.dims.C); + + std::string suffix = bias_str; + if (!impl_selector.empty()) { + if (!suffix.empty()) { + suffix += " "; + } + suffix += "[" + impl_selector + "]"; + } + + std::string name = + make_test_label(prefix, dtype_str, dtype_str, shape, storage_str, suffix); + + test_case.set_name(name); + test_case.set_operator_name("test_etvk.test_conv2d_dw.default"); + + // Input tensor [N, C, H, W] + ValueSpec input( + {config.dims.N, config.dims.C, config.dims.H, config.dims.W}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + + // Weight tensor [C, 1, K_h, K_w] - constant + ValueSpec weight( + {config.dims.C, 1, config.kernel.h, config.kernel.w}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + weight.set_constant(true); + + test_case.add_input_spec(input); + test_case.add_input_spec(weight); + + // Bias (or none) + if (config.has_bias) { + ValueSpec bias( + {config.dims.C}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + bias.set_constant(true); + test_case.add_input_spec(bias); + } else { + ValueSpec none_bias(static_cast(0)); + none_bias.set_none(true); + test_case.add_input_spec(none_bias); + } + + // stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w + test_case.add_input_spec(ValueSpec(static_cast(config.stride.h))); + test_case.add_input_spec(ValueSpec(static_cast(config.stride.w))); + test_case.add_input_spec(ValueSpec(static_cast(config.padding.h))); + test_case.add_input_spec(ValueSpec(static_cast(config.padding.w))); + test_case.add_input_spec(ValueSpec(static_cast(config.dilation.h))); + test_case.add_input_spec(ValueSpec(static_cast(config.dilation.w))); + + // impl_selector string + test_case.add_input_spec(ValueSpec::make_string(impl_selector)); + + // Output tensor [N, C, H_out, W_out] + ValueSpec output( + {config.dims.N, config.dims.C, H_out, W_out}, + dtype, + storage_type, + memory_layout, + DataGenType::ZEROS); + test_case.add_output_spec(output); + + if (dtype == vkapi::kHalf) { + test_case.set_abs_tolerance(1e-1f); + test_case.set_rel_tolerance(1e-2f); + } else { + test_case.set_abs_tolerance(1e-3f); + test_case.set_rel_tolerance(1e-3f); + } + + test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); + + return test_case; +} + +TestCase create_conv2d_pw_test_case( + const Conv2dPwConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout) { + TestCase test_case; + + bool is_perf = config.C_in > kRefDimSizeLimit || + config.C_out > kRefDimSizeLimit || config.H > kRefDimSizeLimit || + config.W > kRefDimSizeLimit; + + std::string prefix = is_perf ? "PERF" : "ACCU"; + std::string storage_str = repr_str(storage_type, memory_layout); + std::string dtype_str = dtype_short(dtype); + std::string bias_str = config.has_bias ? "+bias" : ""; + + // Pointwise conv2d: kernel 1x1, stride 1, pad 0, dilation 1, groups 1 + std::string shape = "[" + std::to_string(config.N) + "," + + std::to_string(config.C_in) + "," + std::to_string(config.H) + "," + + std::to_string(config.W) + "]x[" + std::to_string(config.C_out) + "," + + std::to_string(config.C_in) + ",1,1] s1 p0 d1 g1"; + + std::string name = make_test_label( + prefix, dtype_str, dtype_str, shape, storage_str, bias_str); + + test_case.set_name(name); + test_case.set_operator_name("test_etvk.test_conv2d_pw.default"); + + // Input tensor [N, C_in, H, W] + ValueSpec input( + {config.N, config.C_in, config.H, config.W}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + + // Weight tensor [C_out, C_in, 1, 1] - constant + ValueSpec weight( + {config.C_out, config.C_in, 1, 1}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + weight.set_constant(true); + + test_case.add_input_spec(input); + test_case.add_input_spec(weight); + + // Bias (or none) + if (config.has_bias) { + ValueSpec bias( + {config.C_out}, + dtype, + storage_type, + memory_layout, + DataGenType::RANDOM); + bias.set_constant(true); + test_case.add_input_spec(bias); + } else { + ValueSpec none_bias(static_cast(0)); + none_bias.set_none(true); + test_case.add_input_spec(none_bias); + } + + // impl_selector + ValueSpec impl_selector_spec = ValueSpec::make_string("default"); + test_case.add_input_spec(impl_selector_spec); + + // Output tensor [N, C_out, H, W] + ValueSpec output( + {config.N, config.C_out, config.H, config.W}, + dtype, + storage_type, + memory_layout, + DataGenType::ZEROS); + test_case.add_output_spec(output); + + if (dtype == vkapi::kHalf) { + test_case.set_abs_tolerance(1e-1f); + test_case.set_rel_tolerance(1e-2f); + } else { + test_case.set_abs_tolerance(1e-3f); + test_case.set_rel_tolerance(1e-3f); + } + + test_case.set_shader_filter({"nchw_to", "to_nchw", "view_copy"}); + + return test_case; +} + +int64_t conv2d_flop_calculator(const TestCase& test_case) { + auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); + auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); + auto output_sizes = test_case.outputs()[0].get_tensor_sizes(); + + int64_t N = output_sizes[0]; + int64_t C_out = output_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; + int64_t C_in = input_sizes[1]; + int64_t K_h = weight_sizes[2]; + int64_t K_w = weight_sizes[3]; + + return 2 * N * C_out * C_in * H_out * W_out * K_h * K_w; +} + +int64_t conv2d_dw_flop_calculator(const TestCase& test_case) { + auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); + auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); + auto output_sizes = test_case.outputs()[0].get_tensor_sizes(); + + int64_t N = output_sizes[0]; + int64_t C = output_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; + int64_t K_h = weight_sizes[2]; + int64_t K_w = weight_sizes[3]; + + // Each output element: K_h * K_w multiplies + (K_h * K_w - 1) adds + return 2 * N * C * H_out * W_out * K_h * K_w; +} + +int64_t conv2d_pw_flop_calculator(const TestCase& test_case) { + auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); + auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); + + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t H = input_sizes[2]; + int64_t W = input_sizes[3]; + int64_t C_out = weight_sizes[0]; + + return 2 * N * C_out * C_in * H * W; +} + +// Reference implementation for general conv2d (groups=1). +// +// Supports both FP32 and (small-shape) FP16 inputs. The math is always done in +// float; for FP16 the master input/weight/bias values are dequantized from +// their half storage via get_element(), and the resulting float reference is +// compared against the dequantized GPU output by validate_against_reference(). +// +// FP16 accumulation error grows with K (= K_h * K_w * C_in). For large-K PERF +// shapes the FP32 reference would diverge from the GPU's FP16 accumulation +// enough to trip even the relaxed half tolerance, producing false failures, so +// those are intentionally left timing-only: this function throws +// std::invalid_argument, which execute_test_cases() catches to skip the +// correctness check (ref_computed stays false) while still benchmarking. +void conv2d_reference_impl(TestCase& test_case) { + const ValueSpec& input = test_case.inputs()[0]; + const ValueSpec& weight = test_case.inputs()[1]; + const ValueSpec& bias_spec = test_case.inputs()[2]; + ValueSpec& output = test_case.outputs()[0]; + + if (input.dtype != vkapi::kFloat && input.dtype != vkapi::kHalf) { + throw std::invalid_argument("Reference only supports float and half"); + } + + auto input_sizes = input.get_tensor_sizes(); + auto weight_sizes = weight.get_tensor_sizes(); + auto output_sizes = output.get_tensor_sizes(); + + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t H_in = input_sizes[2]; + int64_t W_in = input_sizes[3]; + int64_t C_out = weight_sizes[0]; + int64_t K_h = weight_sizes[2]; + int64_t K_w = weight_sizes[3]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; + + // For FP16, only compute a reference for small (ACCU) shapes where K is small + // enough that FP32-vs-FP16 accumulation error stays within the half + // tolerance. Large-K PERF half shapes stay timing-only via the throw below. + // The predicate mirrors create_conv2d_test_case's is_perf classification. + if (input.dtype == vkapi::kHalf) { + const bool is_perf = conv2d_is_perf_shape(C_in, C_out, H_in, W_in); + if (is_perf) { + throw std::invalid_argument( + "Half reference skipped for large-K PERF shape (timing-only)"); + } + } + + int64_t stride_h = test_case.inputs()[3].get_int_value(); + int64_t stride_w = test_case.inputs()[4].get_int_value(); + int64_t padding_h = test_case.inputs()[5].get_int_value(); + int64_t padding_w = test_case.inputs()[6].get_int_value(); + int64_t dilation_h = test_case.inputs()[7].get_int_value(); + int64_t dilation_w = test_case.inputs()[8].get_int_value(); + + // get_element() materializes a float regardless of dtype (it dequantizes + // half master data), so the same loop body serves both FP32 and FP16. + auto& ref_data = output.get_ref_float_data(); + ref_data.resize(N * C_out * H_out * W_out, 0.0f); + + for (int64_t n = 0; n < N; ++n) { + for (int64_t co = 0; co < C_out; ++co) { + for (int64_t oh = 0; oh < H_out; ++oh) { + for (int64_t ow = 0; ow < W_out; ++ow) { + float sum = 0.0f; + for (int64_t ci = 0; ci < C_in; ++ci) { + for (int64_t kh = 0; kh < K_h; ++kh) { + for (int64_t kw = 0; kw < K_w; ++kw) { + int64_t ih = oh * stride_h - padding_h + kh * dilation_h; + int64_t iw = ow * stride_w - padding_w + kw * dilation_w; + if (ih >= 0 && ih < H_in && iw >= 0 && iw < W_in) { + float in_val = input.get_element( + n * (C_in * H_in * W_in) + ci * (H_in * W_in) + + ih * W_in + iw); + // weight is [C_out, C_in, K_h, K_w] + float w_val = weight.get_element( + co * (C_in * K_h * K_w) + ci * (K_h * K_w) + kh * K_w + + kw); + sum += in_val * w_val; + } + } + } + } + if (!bias_spec.is_none()) { + sum += bias_spec.get_element(co); + } + ref_data + [n * (C_out * H_out * W_out) + co * (H_out * W_out) + oh * W_out + + ow] = sum; + } + } + } + } +} + +// Reference implementation for depthwise conv2d +void conv2d_dw_reference_impl(TestCase& test_case) { + const ValueSpec& input = test_case.inputs()[0]; + const ValueSpec& weight = test_case.inputs()[1]; + const ValueSpec& bias_spec = test_case.inputs()[2]; + ValueSpec& output = test_case.outputs()[0]; + + if (input.dtype != vkapi::kFloat) { + throw std::invalid_argument("Reference only supports float"); + } + + auto input_sizes = input.get_tensor_sizes(); + auto weight_sizes = weight.get_tensor_sizes(); + auto output_sizes = output.get_tensor_sizes(); + + int64_t N = input_sizes[0]; + int64_t C = input_sizes[1]; + int64_t H_in = input_sizes[2]; + int64_t W_in = input_sizes[3]; + int64_t K_h = weight_sizes[2]; + int64_t K_w = weight_sizes[3]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; + + int64_t stride_h = test_case.inputs()[3].get_int_value(); + int64_t stride_w = test_case.inputs()[4].get_int_value(); + int64_t padding_h = test_case.inputs()[5].get_int_value(); + int64_t padding_w = test_case.inputs()[6].get_int_value(); + int64_t dilation_h = test_case.inputs()[7].get_int_value(); + int64_t dilation_w = test_case.inputs()[8].get_int_value(); + + auto& input_data = input.get_float_data(); + auto& weight_data = weight.get_float_data(); + auto& ref_data = output.get_ref_float_data(); + ref_data.resize(N * C * H_out * W_out, 0.0f); + + for (int64_t n = 0; n < N; ++n) { + for (int64_t c = 0; c < C; ++c) { + for (int64_t oh = 0; oh < H_out; ++oh) { + for (int64_t ow = 0; ow < W_out; ++ow) { + float sum = 0.0f; + for (int64_t kh = 0; kh < K_h; ++kh) { + for (int64_t kw = 0; kw < K_w; ++kw) { + int64_t ih = oh * stride_h - padding_h + kh * dilation_h; + int64_t iw = ow * stride_w - padding_w + kw * dilation_w; + if (ih >= 0 && ih < H_in && iw >= 0 && iw < W_in) { + float in_val = input_data + [n * (C * H_in * W_in) + c * (H_in * W_in) + ih * W_in + + iw]; + // weight is [C, 1, K_h, K_w] + float w_val = weight_data[c * (K_h * K_w) + kh * K_w + kw]; + sum += in_val * w_val; + } + } + } + if (!bias_spec.is_none()) { + auto& bias_data = bias_spec.get_float_data(); + sum += bias_data[c]; + } + ref_data + [n * (C * H_out * W_out) + c * (H_out * W_out) + oh * W_out + + ow] = sum; + } + } + } + } +} + +// Reference implementation: pointwise conv2d is essentially a matmul +// output[n][c_out][h][w] = bias[c_out] + +// sum_over_c_in(input[n][c_in][h][w] * weight[c_out][c_in][0][0]) +void conv2d_pw_reference_impl(TestCase& test_case) { + // input[0], weight[1], bias[2], impl_selector[3] + const ValueSpec& input = test_case.inputs()[0]; + const ValueSpec& weight = test_case.inputs()[1]; + const ValueSpec& bias_spec = test_case.inputs()[2]; + ValueSpec& output = test_case.outputs()[0]; + + if (input.dtype != vkapi::kFloat) { + throw std::invalid_argument("Reference only supports float"); + } + + auto input_sizes = input.get_tensor_sizes(); + auto weight_sizes = weight.get_tensor_sizes(); + + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t H = input_sizes[2]; + int64_t W = input_sizes[3]; + int64_t C_out = weight_sizes[0]; + + auto& input_data = input.get_float_data(); + auto& weight_data = weight.get_float_data(); + auto& ref_data = output.get_ref_float_data(); + ref_data.resize(N * C_out * H * W, 0.0f); + + for (int64_t n = 0; n < N; ++n) { + for (int64_t co = 0; co < C_out; ++co) { + for (int64_t h = 0; h < H; ++h) { + for (int64_t w = 0; w < W; ++w) { + float sum = 0.0f; + for (int64_t ci = 0; ci < C_in; ++ci) { + float in_val = + input_data[n * (C_in * H * W) + ci * (H * W) + h * W + w]; + // weight is [C_out, C_in, 1, 1] + float w_val = weight_data[co * C_in + ci]; + sum += in_val * w_val; + } + if (!bias_spec.is_none()) { + auto& bias_data = bias_spec.get_float_data(); + sum += bias_data[co]; + } + ref_data[n * (C_out * H * W) + co * (H * W) + h * W + w] = sum; + } + } + } + } +} + +} // namespace conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.h b/backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.h new file mode 100644 index 00000000000..858e8b22fd7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv2d/conv2d.h @@ -0,0 +1,101 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv2d { + +constexpr int64_t kRefDimSizeLimit = 64; + +// Conv2d shaders are texture-only and require channels-packed layout +constexpr utils::StorageType kStorageType = utils::kTexture3D; +constexpr utils::GPUMemoryLayout kMemoryLayout = utils::kChannelsPacked; + +struct InputDims { + int64_t N; + int64_t C; + int64_t H; + int64_t W; + + InputDims(int64_t n, int64_t c, int64_t h, int64_t w) + : N(n), C(c), H(h), W(w) {} +}; + +struct Conv2dTestConfig { + InputDims dims; + int64_t C_out; + KernelSize kernel; + Stride stride; + Padding padding; + Dilation dilation; + bool has_bias; +}; + +struct Conv2dDwConfig { + InputDims dims; + KernelSize kernel; + Stride stride; + Padding padding; + Dilation dilation; + bool has_bias; +}; + +struct Conv2dPwConfig { + int64_t N; + int64_t C_in; + int64_t C_out; + int64_t H; + int64_t W; + bool has_bias; +}; + +bool conv2d_is_perf_shape(int64_t C_in, int64_t C_out, int64_t H, int64_t W); + +TestCase create_conv2d_test_case( + const Conv2dTestConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout, + const std::string& impl_selector = ""); + +TestCase create_conv2d_dw_test_case( + const Conv2dDwConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout, + const std::string& impl_selector = ""); + +TestCase create_conv2d_pw_test_case( + const Conv2dPwConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout); + +void conv2d_reference_impl(TestCase& test_case); + +void conv2d_dw_reference_impl(TestCase& test_case); + +void conv2d_pw_reference_impl(TestCase& test_case); + +int64_t conv2d_flop_calculator(const TestCase& test_case); + +int64_t conv2d_dw_flop_calculator(const TestCase& test_case); + +int64_t conv2d_pw_flop_calculator(const TestCase& test_case); + +} // namespace conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/conv2d_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/conv2d_sets.cpp new file mode 100644 index 00000000000..19eba08ae69 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/conv2d_sets.cpp @@ -0,0 +1,342 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv2d { + +namespace { + +struct Conv2dCaseGroup { + std::vector configs; + std::vector dtypes; + std::vector impls; +}; + +std::vector generate_test_cases( + const std::vector& groups) { + std::vector test_cases; + + for (const auto& group : groups) { + for (const auto& config : group.configs) { + for (auto dtype : group.dtypes) { + for (const auto& impl : group.impls) { + test_cases.push_back(create_conv2d_test_case( + config, dtype, kStorageType, kMemoryLayout, impl)); + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("conv2d", "correctness") { + // Accuracy shapes (small enough for float reference validation) + std::vector accuracy_configs = { + // 3x3 stride=1 pad=1 same-channels (the bottleneck pattern in TinyCNN) + {InputDims(1, 8, 8, 8), + 8, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 8, 8, 8), + 8, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + {InputDims(1, 16, 16, 16), + 16, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // 3x3 stride=2 (downsample) with channel expansion + {InputDims(1, 8, 16, 16), + 16, + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + // 3x3 stride=1 with channel reduction + {InputDims(1, 16, 8, 8), + 8, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + // Non-multiple-of-4 channels + {InputDims(1, 11, 8, 8), + 13, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + // 3-channel input (like RGB stem) + {InputDims(1, 3, 16, 16), + 8, + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + }; + + // Small shapes used to exercise each im2col intermediate-storage variant + // (buffer / texture2d / texture3d) deterministically and independently of + // the device's auto-selection. All dims < kRefDimSizeLimit so both the FP32 + // and FP16 references validate them. The first two shapes keep M (= H_out * + // W_out) tiny, so the im2col intermediate [1, K_total, H_out, W_out] is small + // and the forced texture3d variant never exercises the large-M texture3d + // Z-layout the auto-selector actually falls back to. The third shape (48x48) + // is included specifically so the forced im2col_tex3d variant exercises the + // texture3d layout (K4 along Z, many spatial rows) at a non-trivial M, while + // still keeping every dim under kRefDimSizeLimit for reference validation. + std::vector per_variant_configs = { + // 3x3 s1 p1, channels multiple of 4 + {InputDims(1, 16, 16, 16), + 16, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // Non-multiple-of-4 channels exercise the Cin padding path + {InputDims(1, 11, 12, 12), + 13, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + // Larger spatial extent (M = 48*48 = 2304) exercises the texture3d im2col + // layout [1, K_total, H_out, W_out] with K4 along Z at a non-trivial M, + // while all dims stay < kRefDimSizeLimit so both FP32 and FP16 references + // validate it. C_in=16 keeps K = 3*3*16 = 144 (same as the 16x16 case) + // so FP16 accumulation stays within tolerance. + {InputDims(1, 16, 48, 48), + 16, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + }; + + // Implementation variants exercised for every small ACCU shape: + // "" -> heuristic-routed (should_use_conv2d_im2col picks direct on + // Adreno for small c_out, im2col on Mali) + // "im2col" -> forced im2col/GEMM path + // "direct" -> forced direct sliding-window path (conv2d_direct_impl) + // Including "direct" guarantees the direct shader gets reference-checked on + // BOTH devices — without it, Mali would always route "" to im2col and never + // exercise the direct path. + const std::vector impls = {"", "im2col", "direct"}; + // Forced-storage im2col variants for the per-variant ACCU coverage. + const std::vector forced_storage_impls = { + "im2col_buffer", "im2col_tex2d", "im2col_tex3d"}; + + // FP16 small shapes get a real reference check (gated in + // conv2d_reference_impl); we run both dtypes so we catch correctness + // regressions in either path. Large-K half stays timing-only via the + // reference's PERF-shape throw. + const std::vector accu_dtypes = { + vkapi::kFloat, vkapi::kHalf}; + + const std::vector groups = { + // Accuracy test cases for all impls and both dtypes. + {accuracy_configs, accu_dtypes, impls}, + // Per-variant forced-storage ACCU cases (FP32 and FP16) so all three + // im2col intermediate-storage variants get deterministic, + // device-independent, reference-checked coverage at small K. + {per_variant_configs, accu_dtypes, forced_storage_impls}, + }; + return { + generate_test_cases(groups), + conv2d_reference_impl, + conv2d_flop_calculator, + /*warmup_runs = */ 5, + /*benchmark_runs = */ 20}; +} + +REGISTER_TEST_CASE_SET("conv2d", "performance") { + // TinyCNN depth estimator hotspots (from profiling + // UNTRAINED_TinyCNNDepthEstimatorRealTime_Vulkan.pte). + // Each entry lists (C_in, H, W) -> C_out, all 3x3 stride=1 pad=1 unless + // noted. Together the first 6 entries account for ~89% of all conv time. + std::vector perf_configs = { + // #1: 21.25% — (1,128,36,48)->(1,128,36,48) + {InputDims(1, 128, 36, 48), + 128, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // #2: 20.68% — (1,256,18,24)->(1,256,18,24) + {InputDims(1, 256, 18, 24), + 256, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // #3: 20.01% — (1,64,72,96)->(1,64,72,96) + {InputDims(1, 64, 72, 96), + 64, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // #4: 13.25% — (1,32,144,192)->(1,32,144,192) + {InputDims(1, 32, 144, 192), + 32, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // #5: 6.74% — (1,64,36,48)->(1,64,36,48) + {InputDims(1, 64, 36, 48), + 64, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // #6: 5.90% — (1,32,72,96)->(1,32,72,96) + {InputDims(1, 32, 72, 96), + 32, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // Secondary cases + // 3x3 stride=2 downsample with channel expansion: 1.52% + {InputDims(1, 32, 72, 96), + 128, + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + // 3x3 stride=1 same-shape, smaller spatial: 1.51% + {InputDims(1, 128, 18, 24), + 128, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + // 3x3 stride=1, channel reduction + {InputDims(1, 128, 18, 24), + 64, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 64, 36, 48), + 32, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + // 3x3 stride=2 downsample, same channels + {InputDims(1, 32, 72, 96), + 32, + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 64, 36, 48), + 64, + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + // RGB stem + {InputDims(1, 3, 144, 192), + 32, + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + }; + + // Boundary pair straddling the should_use_conv2d_im2col() c_out >= 128 + // routing threshold. Spatial dims are tiny (8x8) so the FP32 float reference + // stays cheap, but c_out = 64 / 128 are both >= kRefDimSizeLimit, so these + // get the PERF label. FP32 PERF cases are still numerically VERIFIED (the + // reference's invalid_argument throw that skips the check only fires for + // half), so both implementations are cross-checked against the float + // reference at the boundary. Run all three impls: at c_out = 64 the heuristic + // ("") picks direct on Adreno / im2col on Mali; at c_out = 128 it picks + // im2col on both — and "direct"/"im2col" force each path regardless, proving + // the two implementations agree at the boundary on either device. + std::vector boundary_configs = { + // c_out = 64 (< 128): below the threshold + {InputDims(1, 16, 8, 8), + 64, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + // c_out = 128 (== 128): at/above the threshold + {InputDims(1, 16, 8, 8), + 128, + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + }; + + const std::vector impls = {"", "im2col", "direct"}; + + const std::vector groups = { + // The c_out boundary pair (FP32 only) through all three impls. FP32 PERF + // cases are reference-VERIFIED, so the direct and im2col paths are both + // cross-checked against the float reference at the routing threshold. + {boundary_configs, {vkapi::kFloat}, impls}, + // Performance test cases (float and half) for all impls. + {perf_configs, {vkapi::kFloat, vkapi::kHalf}, impls}, + }; + return { + generate_test_cases(groups), + conv2d_reference_impl, + conv2d_flop_calculator, + /*warmup_runs = */ 5, + /*benchmark_runs = */ 20}; +} + +} // namespace conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/dw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/dw_sets.cpp new file mode 100644 index 00000000000..1b71c730164 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/dw_sets.cpp @@ -0,0 +1,189 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv2d { + +namespace { + +std::vector generate_dw_test_cases( + const std::vector& configs, + const std::vector& dtypes, + const std::vector& impls) { + std::vector test_cases; + + for (const auto& config : configs) { + for (auto dtype : dtypes) { + for (const auto& impl : impls) { + test_cases.push_back(create_conv2d_dw_test_case( + config, dtype, kStorageType, kMemoryLayout, impl)); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("conv2d", "dw_correctness") { + // Accuracy shapes (small enough for float reference validation) + std::vector accuracy_configs = { + {InputDims(1, 8, 16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 8, 16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + true}, + {InputDims(1, 8, 16, 16), + KernelSize(5, 5), + Stride(1, 1), + Padding(2, 2), + Dilation(1, 1), + false}, + {InputDims(1, 8, 16, 16), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + // Non-multiple-of-4 channels + {InputDims(1, 11, 16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 3, 16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + }; + + // Accuracy test cases (float only, auto-selected implementation) + return { + generate_dw_test_cases(accuracy_configs, {vkapi::kFloat}, {""}), + conv2d_dw_reference_impl, + conv2d_dw_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("conv2d", "dw_performance") { + // EdgeTAM depthwise shapes (from profiling data) + std::vector perf_configs = { + // Backbone stem and early stages + {InputDims(1, 24, 512, 512), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 48, 256, 256), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 48, 256, 256), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 96, 128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 96, 128, 128), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 192, 64, 64), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 192, 64, 64), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 384, 32, 32), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + // 5x5 kernels + {InputDims(1, 48, 256, 256), + KernelSize(5, 5), + Stride(1, 1), + Padding(2, 2), + Dilation(1, 1), + false}, + {InputDims(1, 96, 128, 128), + KernelSize(5, 5), + Stride(1, 1), + Padding(2, 2), + Dilation(1, 1), + false}, + // FPN/Neck + {InputDims(1, 256, 256, 256), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + {InputDims(1, 256, 128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + false}, + }; + + const std::vector impls = { + // Auto-selection (empty impl_selector) + "", + // Force b4x2 variant + "b4x2", + // Force b1x1 variant (only for 3x3 kernels; for 5x5 it falls back + // to default, but we still generate it to test the fallback path) + "b1x1", + }; + + // Performance test cases (float and half) + return { + generate_dw_test_cases( + perf_configs, {vkapi::kFloat, vkapi::kHalf}, impls), + conv2d_dw_reference_impl, + conv2d_dw_flop_calculator}; +} + +} // namespace conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/pw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/pw_sets.cpp new file mode 100644 index 00000000000..afb206ea6e2 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/conv2d/test_sets/pw_sets.cpp @@ -0,0 +1,104 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace conv2d { + +namespace { + +struct Conv2dPwCaseGroup { + std::vector configs; + std::vector dtypes; +}; + +std::vector generate_pw_test_cases( + const std::vector& groups) { + std::vector test_cases; + + for (const auto& group : groups) { + for (const auto& config : group.configs) { + for (auto dtype : group.dtypes) { + test_cases.push_back(create_conv2d_pw_test_case( + config, dtype, kStorageType, kMemoryLayout)); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("conv2d", "pw_correctness") { + // Accuracy shapes (small enough for float reference validation) + std::vector accuracy_configs = { + {1, 16, 32, 8, 8, false}, + {1, 32, 16, 8, 8, false}, + {1, 16, 32, 8, 8, true}, + // Non-multiple-of-4 channels + {1, 13, 27, 8, 8, false}, + {1, 33, 17, 8, 8, false}, + }; + + const std::vector groups = { + // Accuracy test cases (float only) + {accuracy_configs, {vkapi::kFloat}}, + }; + return { + generate_pw_test_cases(groups), + conv2d_pw_reference_impl, + conv2d_pw_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("conv2d", "pw_performance") { + // Accuracy shapes (small enough for float reference validation) that + // create_conv2d_pw_test_case labels PERF, since a channel count exceeds + // kRefDimSizeLimit + std::vector accuracy_configs = { + {1, 48, 96, 16, 16, false}, + {1, 96, 48, 16, 16, false}, + }; + + // EdgeTAM performance shapes + std::vector perf_configs = { + // EdgeTAM backbone stages + {1, 48, 96, 256, 256, false}, + {1, 96, 48, 256, 256, false}, + {1, 96, 192, 128, 128, false}, + {1, 192, 96, 128, 128, false}, + {1, 192, 384, 64, 64, false}, + {1, 384, 192, 64, 64, false}, + {1, 384, 768, 32, 32, false}, + {1, 768, 384, 32, 32, false}, + // EdgeTAM FPN/Neck + {1, 48, 256, 256, 256, false}, + {1, 256, 32, 256, 256, false}, + {1, 96, 256, 128, 128, false}, + {1, 256, 64, 128, 128, false}, + }; + + const std::vector groups = { + // Accuracy test cases (float only) + {accuracy_configs, {vkapi::kFloat}}, + // Performance test cases (float and half) + {perf_configs, {vkapi::kFloat, vkapi::kHalf}}, + }; + return { + generate_pw_test_cases(groups), + conv2d_pw_reference_impl, + conv2d_pw_flop_calculator}; +} + +} // namespace conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.cpp b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.cpp new file mode 100644 index 00000000000..84454d8723d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.cpp @@ -0,0 +1,195 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace embedding_q4gsw { + +TestCase create_test_case(const EmbeddingConfig& config) { + TestCase test_case; + + // Compute the output shape for label: indices_shape + [embed_dim] + std::vector output_shape_for_label = config.indices_shape; + output_shape_for_label.push_back(config.embed_dim); + + // Treat any case that uses the large llama vocab/embed sizes as PERF. + // Otherwise default to ACCU (no PERF distinction is exercised by this op). + bool is_perf = config.vocab_size > 1024 || config.embed_dim > 1024; + std::string prefix = is_perf ? "PERF" : "ACCU"; + + std::string in_dtype = dtype_short(vkapi::kInt); // indices are i32 + std::string out_dtype = dtype_short(config.dtype); + + std::string storage_str = repr_str(config.storage_type, utils::kWidthPacked); + + std::string shape_str = shape_bracket(config.indices_shape) + "x[" + + std::to_string(config.vocab_size) + "," + + std::to_string(config.embed_dim) + "]"; + shape_str += " g" + std::to_string(config.group_size); + if (config.is_linear_weight) { + shape_str += " lw"; + } + std::string suffix; + if (config.scales_dtype == vkapi::kFloat) { + suffix = "[f32_scales]"; + } + std::string name = make_test_label( + prefix, in_dtype, out_dtype, shape_str, storage_str, suffix); + test_case.set_name(name); + test_case.set_operator_name("et_vk.embedding_q4gsw.default"); + test_case.set_shader_filter({}); + + // Weight: [vocab_size, embed_dim / 2] packed uint8 + ValueSpec weight( + {config.vocab_size, config.embed_dim / 2}, + vkapi::kByte, + utils::kBuffer, + utils::kWidthPacked, + DataGenType::RANDINT4); + weight.set_constant(true); + test_case.add_input_spec(weight); + + // Weight scales: [vocab_size, groups_per_row] + int64_t groups_per_row = config.embed_dim / config.group_size; + ValueSpec weight_scales( + {config.vocab_size, groups_per_row}, + config.scales_dtype, + utils::kBuffer, + utils::kWidthPacked, + DataGenType::RANDOM_SCALES); + weight_scales.set_constant(true); + test_case.add_input_spec(weight_scales); + + // Group size: int scalar + ValueSpec group_size_spec(static_cast(config.group_size)); + test_case.add_input_spec(group_size_spec); + + // Indices: [batch, seq_len] int32 + ValueSpec indices( + config.indices_shape, + vkapi::kInt, + config.storage_type, + utils::kWidthPacked, + DataGenType::RANDINT); + + // Clamp indices to valid vocab range + indices.ensure_data_generated(); + for (auto& idx : indices.get_int32_data()) { + idx = std::abs(idx) % config.vocab_size; + } + + test_case.add_input_spec(indices); + + // is_linear_weight: bool scalar + ValueSpec is_linear_weight_spec(config.is_linear_weight); + test_case.add_input_spec(is_linear_weight_spec); + + // Output: indices.shape + [embed_dim] + std::vector output_shape = config.indices_shape; + output_shape.push_back(config.embed_dim); + ValueSpec output( + output_shape, config.dtype, config.storage_type, utils::kWidthPacked); + test_case.add_output_spec(output); + + return test_case; +} + +// CPU reference: unpack 4-bit weights, dequantize, and perform embedding lookup +void embedding_4bit_reference(TestCase& tc) { + auto& weight_spec = tc.inputs()[0]; + auto& scales_spec = tc.inputs()[1]; + int32_t group_size = tc.inputs()[2].get_int_value(); + auto& indices_spec = tc.inputs()[3]; + bool is_linear_weight = tc.inputs()[4].get_bool_value(); + auto& output_spec = tc.outputs()[0]; + + weight_spec.ensure_data_generated(); + scales_spec.ensure_data_generated(); + indices_spec.ensure_data_generated(); + + const auto& weight_data = weight_spec.get_uint8_data(); + const auto& indices_data = indices_spec.get_int32_data(); + + bool scales_are_half = (scales_spec.dtype == vkapi::kHalf); + + int64_t packed_dim = weight_spec.sizes[1]; + int64_t embed_dim = packed_dim * 2; + int64_t groups_per_row = scales_spec.sizes[1]; + + int64_t num_indices = 1; + for (auto s : indices_spec.sizes) { + num_indices *= s; + } + + int64_t total_output = num_indices * embed_dim; + + // Always populate ref_float_data so the caching framework can distribute it + output_spec.get_ref_float_data().resize(total_output); + + bool output_is_half = (output_spec.dtype == vkapi::kHalf); + if (output_is_half) { + output_spec.get_ref_half_data().resize(total_output); + } + + for (int64_t i = 0; i < num_indices; ++i) { + int32_t idx = indices_data[i]; + for (int64_t d = 0; d < embed_dim; ++d) { + int64_t packed_idx = d / 2; + uint8_t packed_byte = weight_data[idx * packed_dim + packed_idx]; + + // Unpack: packed_byte = (even_val + 8) << 4 | (odd_val + 8) + // Even d -> high nibble, odd d -> low nibble + // For linear weight packing, nibble order is swapped + int int4_val; + if (d % 2 == 0) { + if (is_linear_weight) { + int4_val = static_cast(packed_byte & 0xF) - 8; + } else { + int4_val = static_cast(packed_byte >> 4) - 8; + } + } else { + if (is_linear_weight) { + int4_val = static_cast(packed_byte >> 4) - 8; + } else { + int4_val = static_cast(packed_byte & 0xF) - 8; + } + } + + int64_t group_idx = d / group_size; + int64_t scale_idx = idx * groups_per_row + group_idx; + + float scale; + if (scales_are_half) { + uint16_t scale_half = scales_spec.get_half_data()[scale_idx]; + scale = half_to_float(scale_half); + } else { + scale = scales_spec.get_float_data()[scale_idx]; + } + + float result = static_cast(int4_val) * scale; + + // Always store float reference + output_spec.get_ref_float_data()[i * embed_dim + d] = result; + + if (output_is_half) { + output_spec.get_ref_half_data()[i * embed_dim + d] = + float_to_half(result); + } + } + } +} + +} // namespace embedding_q4gsw +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.h b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.h new file mode 100644 index 00000000000..0a2103350a8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/embedding_q4gsw.h @@ -0,0 +1,39 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace embedding_q4gsw { + +struct EmbeddingConfig { + int64_t vocab_size; + int64_t embed_dim; + int64_t group_size; + std::vector indices_shape; + std::string test_case_name = "placeholder"; + vkapi::ScalarType dtype = vkapi::kHalf; + vkapi::ScalarType scales_dtype = vkapi::kHalf; + utils::StorageType storage_type = utils::kBuffer; + bool is_linear_weight = false; +}; + +TestCase create_test_case(const EmbeddingConfig& config); + +void embedding_4bit_reference(TestCase& tc); + +} // namespace embedding_q4gsw +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_sets/embedding_q4gsw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_sets/embedding_q4gsw_sets.cpp new file mode 100644 index 00000000000..614eeb721a7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/embedding_q4gsw/test_sets/embedding_q4gsw_sets.cpp @@ -0,0 +1,328 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace embedding_q4gsw { + +namespace { + +std::vector generate_test_cases( + const std::vector& configs) { + std::vector test_cases; + test_cases.reserve(configs.size()); + for (const auto& config : configs) { + test_cases.push_back(create_test_case(config)); + } + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("embedding_q4gsw", "correctness") { + const std::vector configs = { + // --- is_linear_weight = true --- + + // Basic test with linear weight packing + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_linear_weight", + .is_linear_weight = true}, + + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d_linear_weight", + .is_linear_weight = true}, + + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup_linear_weight", + .is_linear_weight = true}, + + // fp32 output with linear weight + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_fp32_linear_weight", + .dtype = vkapi::kFloat, + .is_linear_weight = true}, + + // Texture3D with linear weight + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_texture_linear_weight", + .storage_type = utils::kTexture3D, + .is_linear_weight = true}, + + // --- Half scales (default) --- + + // Basic test: small vocab, small embed_dim + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d"}, + + // 2D indices + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d"}, + + // Larger vocab, multiple groups + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup"}, + + // group_size == embed_dim (single group) + {.vocab_size = 50, + .embed_dim = 64, + .group_size = 64, + .indices_shape = {2, 4}, + .test_case_name = "single_group"}, + + // fp32 output variants (half scales) + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_fp32", + .dtype = vkapi::kFloat}, + + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d_fp32", + .dtype = vkapi::kFloat}, + + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup_fp32", + .dtype = vkapi::kFloat}, + + {.vocab_size = 50, + .embed_dim = 64, + .group_size = 64, + .indices_shape = {2, 4}, + .test_case_name = "single_group_fp32", + .dtype = vkapi::kFloat}, + + // Texture3D variants (fp16 output, half scales) + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_texture", + .dtype = vkapi::kHalf, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d_texture", + .dtype = vkapi::kHalf, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup_texture", + .dtype = vkapi::kHalf, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 50, + .embed_dim = 64, + .group_size = 64, + .indices_shape = {2, 4}, + .test_case_name = "single_group_texture", + .dtype = vkapi::kHalf, + .storage_type = utils::kTexture3D}, + + // Texture3D variants (fp32 output, half scales) + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_fp32_texture", + .dtype = vkapi::kFloat, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d_fp32_texture", + .dtype = vkapi::kFloat, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup_fp32_texture", + .dtype = vkapi::kFloat, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 50, + .embed_dim = 64, + .group_size = 64, + .indices_shape = {2, 4}, + .test_case_name = "single_group_fp32_texture", + .dtype = vkapi::kFloat, + .storage_type = utils::kTexture3D}, + + // --- Float scales --- + + // Buffer variants with float scales + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_float_scales", + .dtype = vkapi::kHalf, + .scales_dtype = vkapi::kFloat}, + + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d_float_scales", + .dtype = vkapi::kHalf, + .scales_dtype = vkapi::kFloat}, + + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup_float_scales", + .dtype = vkapi::kHalf, + .scales_dtype = vkapi::kFloat}, + + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_fp32_float_scales", + .dtype = vkapi::kFloat, + .scales_dtype = vkapi::kFloat}, + + {.vocab_size = 32, + .embed_dim = 64, + .group_size = 32, + .indices_shape = {2, 3}, + .test_case_name = "small_2d_fp32_float_scales", + .dtype = vkapi::kFloat, + .scales_dtype = vkapi::kFloat}, + + {.vocab_size = 100, + .embed_dim = 128, + .group_size = 32, + .indices_shape = {4, 8}, + .test_case_name = "medium_multigroup_fp32_float_scales", + .dtype = vkapi::kFloat, + .scales_dtype = vkapi::kFloat}, + + // Texture3D with float scales + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_float_scales_texture", + .dtype = vkapi::kHalf, + .scales_dtype = vkapi::kFloat, + .storage_type = utils::kTexture3D}, + + {.vocab_size = 16, + .embed_dim = 32, + .group_size = 32, + .indices_shape = {4}, + .test_case_name = "small_1d_fp32_float_scales_texture", + .dtype = vkapi::kFloat, + .scales_dtype = vkapi::kFloat, + .storage_type = utils::kTexture3D}, + }; + return {generate_test_cases(configs), embedding_4bit_reference}; +} + +REGISTER_TEST_CASE_SET("embedding_q4gsw", "performance") { + const std::vector configs = { + // Llama 3.2 1B with linear weight packing + {.vocab_size = 128256, + .embed_dim = 2048, + .group_size = 32, + .indices_shape = {1, 2047}, + .test_case_name = "llama_3_2_1b_prefill_fp32_linear_weight", + .dtype = vkapi::kFloat, + .storage_type = utils::kBuffer, + .is_linear_weight = true}, + + // Llama 3.2 1B prefill configuration (fp32 output, half scales) + {.vocab_size = 128256, + .embed_dim = 2048, + .group_size = 32, + .indices_shape = {1, 2047}, + .test_case_name = "llama_3_2_1b_prefill_fp32", + .dtype = vkapi::kFloat, + .storage_type = utils::kBuffer}, + + // Llama 3.2 1B prefill configuration (fp32 output, float scales) + {.vocab_size = 128256, + .embed_dim = 2048, + .group_size = 32, + .indices_shape = {1, 2047}, + .test_case_name = "llama_3_2_1b_prefill_fp32_float_scales", + .dtype = vkapi::kFloat, + .scales_dtype = vkapi::kFloat, + .storage_type = utils::kBuffer}, + + // Llama 3.2 1B prefill configuration (fp16 output, half scales) + {.vocab_size = 128256, + .embed_dim = 2048, + .group_size = 32, + .indices_shape = {1, 2047}, + .test_case_name = "llama_3_2_1b_prefill_fp16", + .dtype = vkapi::kHalf, + .storage_type = utils::kBuffer}, + + // Llama 3.2 1B prefill configuration (fp16 output, float scales) + {.vocab_size = 128256, + .embed_dim = 2048, + .group_size = 32, + .indices_shape = {1, 2047}, + .test_case_name = "llama_3_2_1b_prefill_fp16_float_scales", + .dtype = vkapi::kHalf, + .scales_dtype = vkapi::kFloat, + .storage_type = utils::kBuffer}, + }; + return {generate_test_cases(configs), embedding_4bit_reference}; +} + +} // namespace embedding_q4gsw +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_mm.cpp b/backends/vulkan/test/ops/benchmark/cases/mm/mm.cpp similarity index 68% rename from backends/vulkan/test/custom_ops/test_mm.cpp rename to backends/vulkan/test/ops/benchmark/cases/mm/mm.cpp index 4a8d9de49b6..0526eac1e42 100644 --- a/backends/vulkan/test/custom_ops/test_mm.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/mm/mm.cpp @@ -4,58 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include - -#include -#include - -#include "utils.h" +#include -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 256; +#include +#include +#include -// Return the input data as float regardless of dtype. For kHalf, the -// underlying half_data is converted via half_to_float; for kFloat the -// existing float_data is copied. The CPU reference computes in float and -// the framework compares against (and converts back to) the GPU output. -static std::vector as_float_data(const ValueSpec& spec) { - if (spec.dtype == vkapi::kFloat) { - return spec.get_float_data(); - } - if (spec.dtype == vkapi::kHalf) { - const auto& half_bits = spec.get_half_data(); - std::vector out(half_bits.size()); - for (size_t i = 0; i < half_bits.size(); ++i) { - out[i] = half_to_float(half_bits[i]); - } - return out; - } - throw std::invalid_argument("as_float_data: unsupported dtype"); -} +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace mm { -struct MmConfig { - // mat1: [M, K] or [B, M, K] - // mat2: [K, N] or [B, K, N] - int64_t B; // batch size, 0 for non-batched - int64_t M; - int64_t K; - int64_t N; - bool has_bias; // true for addmm/linear - bool mat2_is_transposed; // true for linear (weight is [N, K]) - bool mat2_is_constant; // true to test prepacked linear path - // "default" routes through aten.{mm,linear,...}.default (production path). - // "coopmat" / "tiled" force-dispatch a specific shader implementation. - std::string impl_selector = "default"; -}; - -struct MmShape { - int64_t B, M, K, N; -}; - -static TestCase create_mm_test_case( +TestCase create_mm_test_case( const MmConfig& config, vkapi::ScalarType dtype, utils::StorageType storage_type, @@ -67,7 +27,7 @@ static TestCase create_mm_test_case( config.N > kRefDimSizeLimit; std::string prefix = is_perf ? "PERF" : "ACCU"; - std::string storage_str = repr_str(storage_type, memory_layout); + std::string storage_str = repr_str(storage_type, utils::kWidthPacked); std::string dtype_str = dtype_short(dtype); // Determine op type string @@ -253,12 +213,73 @@ static TestCase create_mm_test_case( return test_case; } +int64_t mm_flop_calculator(const TestCase& test_case) { + const auto& out_sizes = test_case.outputs()[0].get_tensor_sizes(); + const std::string& op_name = test_case.operator_name(); + + int64_t M, N, K; + + if (op_name == "test_etvk.test_mm.default") { + // mat1[0], mat2[1], impl_selector[2] + auto mat1_sizes = test_case.inputs()[0].get_tensor_sizes(); + M = mat1_sizes[0]; + K = mat1_sizes[1]; + N = out_sizes[1]; + return 2 * M * K * N; + } else if (op_name == "test_etvk.test_bmm.default") { + // mat1[0], mat2[1], impl_selector[2] + auto mat1_sizes = test_case.inputs()[0].get_tensor_sizes(); + int64_t B = mat1_sizes[0]; + M = mat1_sizes[1]; + K = mat1_sizes[2]; + N = out_sizes[2]; + return 2 * B * M * K * N; + } else if (op_name == "test_etvk.test_addmm.default") { + // self[0], mat1[1], mat2[2], beta[3], alpha[4], impl_selector[5] + auto mat1_sizes = test_case.inputs()[1].get_tensor_sizes(); + M = mat1_sizes[0]; + K = mat1_sizes[1]; + N = out_sizes[1]; + return 2 * M * K * N; + } else if (op_name == "test_etvk.test_linear.default") { + // input[0], weight[1], bias[2], impl_selector[3] + auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); + auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); + M = input_sizes[0]; + K = input_sizes[1]; + N = weight_sizes[0]; + return 2 * M * K * N; + } + return 1; +} + +namespace { + +// Return the input data as float regardless of dtype. For kHalf, the +// underlying half_data is converted via half_to_float; for kFloat the +// existing float_data is copied. The CPU reference computes in float and +// the framework compares against (and converts back to) the GPU output. +std::vector as_float_data(const ValueSpec& spec) { + if (spec.dtype == vkapi::kFloat) { + return spec.get_float_data(); + } + if (spec.dtype == vkapi::kHalf) { + const auto& half_bits = spec.get_half_data(); + std::vector out(half_bits.size()); + for (size_t i = 0; i < half_bits.size(); ++i) { + out[i] = half_to_float(half_bits[i]); + } + return out; + } + throw std::invalid_argument("as_float_data: unsupported dtype"); +} + // Reference implementation for mm/bmm // Input layout per test operator: // test_mm/test_bmm: mat1[0], mat2[1], impl_selector[2] // test_addmm: self[0], mat1[1], mat2[2], beta[3], alpha[4], impl_selector[5] // test_linear: input[0], weight[1], bias[2], impl_selector[3] -static void mm_reference_impl(TestCase& test_case) { +void mm_reference_impl(TestCase& test_case) { const std::string& op_name = test_case.operator_name(); ValueSpec& output = test_case.outputs()[0]; auto out_sizes = output.get_tensor_sizes(); @@ -390,189 +411,13 @@ static void mm_reference_impl(TestCase& test_case) { } } -static std::vector generate_mm_test_cases() { - std::vector test_cases; - - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - std::vector shapes = { - // Accuracy shapes (float) - {0, 64, 128, 64}, - {0, 128, 256, 128}, - {0, 32, 64, 256}, - {1, 64, 128, 64}, - {1, 4, 32, 16}, - // Non-multiple-of-4 accuracy shapes (exercises scalar shader fallback) - {0, 57, 131, 43}, - {0, 33, 67, 91}, - {1, 19, 53, 37}, - {0, 64, 128, 47}, // only N unaligned - {0, 64, 47, 128}, // only K unaligned - // Performance shapes (half) - {0, 4096, 1024, 256}, - {0, 4096, 256, 128}, - {0, 4096, 128, 256}, - {1, 4096, 256, 1024}, - {1, 4096, 256, 128}, - {1, 256, 4096, 64}, - {0, 4096, 64, 128}, - }; - - // Coopmat shader requires M%64==0, N%64==0, K%32==0 (no partial-tile or - // K-tail handling). Sweep both "coopmat" and "tiled" force-dispatch variants - // for shapes that satisfy alignment, on buffer storage only (coopmat - // requires buffer outputs). - auto coopmat_shape_eligible = [](const MmShape& s) { - return s.B == 0 && s.M % 64 == 0 && s.N % 64 == 0 && s.K % 32 == 0; - }; - - // The "coopmat" forced-dispatch sweep skips the runtime eligibility check - // (it's intentionally bypassing the gate to exercise the shader). Skip - // generating those cases on adapters that can't actually run the shader, - // or pipeline creation/dispatch will fail. - const auto* adapter = api::context()->adapter_ptr(); - const bool coopmat_runnable = - adapter->supports_cooperative_matrix() && adapter->subgroup_size() == 64; - std::vector coopmat_sweep_selectors = {"tiled"}; - if (coopmat_runnable) { - coopmat_sweep_selectors.push_back("coopmat"); - } - - for (const auto& s : shapes) { - bool is_batched = s.B > 0; - bool is_perf = s.M > kRefDimSizeLimit || s.K > kRefDimSizeLimit || - s.N > kRefDimSizeLimit; - - std::vector dtypes = is_perf - ? std::vector{vkapi::kFloat, vkapi::kHalf} - : std::vector{vkapi::kFloat}; - - MmConfig dynamic_cfg{s.B, s.M, s.K, s.N, false, false, false}; - MmConfig const_cfg{s.B, s.M, s.K, s.N, false, false, true}; - - for (auto dtype : dtypes) { - for (auto st : storage_types) { - test_cases.push_back( - create_mm_test_case(dynamic_cfg, dtype, st, utils::kWidthPacked)); - test_cases.push_back( - create_mm_test_case(const_cfg, dtype, st, utils::kWidthPacked)); - } - - // Coopmat A/B sweep: only on aligned shapes + buffer storage. - if (coopmat_shape_eligible(s)) { - for (const auto& sel : coopmat_sweep_selectors) { - MmConfig dyn = dynamic_cfg; - dyn.impl_selector = sel; - test_cases.push_back(create_mm_test_case( - dyn, dtype, utils::kBuffer, utils::kWidthPacked)); - MmConfig con = const_cfg; - con.impl_selector = sel; - test_cases.push_back(create_mm_test_case( - con, dtype, utils::kBuffer, utils::kWidthPacked)); - } - } - - if (!is_batched) { - MmConfig addmm_cfg{s.B, s.M, s.K, s.N, true, false, false}; - MmConfig addmm_const_cfg{s.B, s.M, s.K, s.N, true, false, true}; - MmConfig linear_cfg{s.B, s.M, s.K, s.N, false, true, false}; - MmConfig linear_bias_cfg{s.B, s.M, s.K, s.N, true, true, false}; - - for (auto st : storage_types) { - test_cases.push_back( - create_mm_test_case(addmm_cfg, dtype, st, utils::kWidthPacked)); - test_cases.push_back(create_mm_test_case( - addmm_const_cfg, dtype, st, utils::kWidthPacked)); - test_cases.push_back( - create_mm_test_case(linear_cfg, dtype, st, utils::kWidthPacked)); - test_cases.push_back(create_mm_test_case( - linear_bias_cfg, dtype, st, utils::kWidthPacked)); - } - - // Coopmat A/B sweep on linear paths too (only aligned shapes). - if (coopmat_shape_eligible(s)) { - for (const auto& sel : coopmat_sweep_selectors) { - MmConfig lin = linear_cfg; - lin.impl_selector = sel; - test_cases.push_back(create_mm_test_case( - lin, dtype, utils::kBuffer, utils::kWidthPacked)); - MmConfig lin_bias = linear_bias_cfg; - lin_bias.impl_selector = sel; - test_cases.push_back(create_mm_test_case( - lin_bias, dtype, utils::kBuffer, utils::kWidthPacked)); - } - } - } - } - } - - return test_cases; -} - -static int64_t mm_flop_calculator(const TestCase& test_case) { - const auto& out_sizes = test_case.outputs()[0].get_tensor_sizes(); - const std::string& op_name = test_case.operator_name(); - - int64_t M, N, K; +} // namespace - if (op_name == "test_etvk.test_mm.default") { - // mat1[0], mat2[1], impl_selector[2] - auto mat1_sizes = test_case.inputs()[0].get_tensor_sizes(); - M = mat1_sizes[0]; - K = mat1_sizes[1]; - N = out_sizes[1]; - return 2 * M * K * N; - } else if (op_name == "test_etvk.test_bmm.default") { - // mat1[0], mat2[1], impl_selector[2] - auto mat1_sizes = test_case.inputs()[0].get_tensor_sizes(); - int64_t B = mat1_sizes[0]; - M = mat1_sizes[1]; - K = mat1_sizes[2]; - N = out_sizes[2]; - return 2 * B * M * K * N; - } else if (op_name == "test_etvk.test_addmm.default") { - // self[0], mat1[1], mat2[2], beta[3], alpha[4], impl_selector[5] - auto mat1_sizes = test_case.inputs()[1].get_tensor_sizes(); - M = mat1_sizes[0]; - K = mat1_sizes[1]; - N = out_sizes[1]; - return 2 * M * K * N; - } else if (op_name == "test_etvk.test_linear.default") { - // input[0], weight[1], bias[2], impl_selector[3] - auto input_sizes = test_case.inputs()[0].get_tensor_sizes(); - auto weight_sizes = test_case.inputs()[1].get_tensor_sizes(); - M = input_sizes[0]; - K = input_sizes[1]; - N = weight_sizes[0]; - return 2 * M * K * N; - } - return 1; -} - -static void reference_impl(TestCase& test_case) { +void reference_impl(TestCase& test_case) { mm_reference_impl(test_case); } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Matrix Multiply (mm/bmm/addmm/linear) Benchmark" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_mm_test_cases, - mm_flop_calculator, - "MatMul", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace mm +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/mm/mm.h b/backends/vulkan/test/ops/benchmark/cases/mm/mm.h new file mode 100644 index 00000000000..b4ec8f21795 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/mm/mm.h @@ -0,0 +1,49 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace mm { + +constexpr int64_t kRefDimSizeLimit = 256; + +struct MmConfig { + // mat1: [M, K] or [B, M, K] + // mat2: [K, N] or [B, K, N] + int64_t B; // batch size, 0 for non-batched + int64_t M; + int64_t K; + int64_t N; + bool has_bias; // true for addmm/linear + bool mat2_is_transposed; // true for linear (weight is [N, K]) + bool mat2_is_constant; // true to test prepacked linear path + // "default" routes through aten.{mm,linear,...}.default (production path). + // "coopmat" / "tiled" force-dispatch a specific shader implementation. + std::string impl_selector = "default"; +}; + +TestCase create_mm_test_case( + const MmConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout); + +void reference_impl(TestCase& test_case); + +int64_t mm_flop_calculator(const TestCase& test_case); + +} // namespace mm +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/mm/test_sets/mm_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/mm/test_sets/mm_sets.cpp new file mode 100644 index 00000000000..232e93d326f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/mm/test_sets/mm_sets.cpp @@ -0,0 +1,171 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace mm { + +namespace { + +struct MmShape { + int64_t B, M, K, N; +}; + +// Coopmat shader requires M%64==0, N%64==0, K%32==0 (no partial-tile or K-tail +// handling). The force-dispatch selector sweep runs only for shapes that +// satisfy alignment, on buffer storage only (coopmat requires buffer outputs). +bool coopmat_shape_eligible(const MmShape& s) { + return s.B == 0 && s.M % 64 == 0 && s.N % 64 == 0 && s.K % 32 == 0; +} + +std::vector generate_mm_test_cases( + const std::vector& shapes, + const std::vector& dtypes, + const std::vector& storage_types, + const std::vector& coopmat_sweep_selectors) { + std::vector test_cases; + + // The "coopmat" forced-dispatch sweep skips the runtime eligibility check + // (it's intentionally bypassing the gate to exercise the shader). Skip + // generating those cases on adapters that can't actually run the shader, + // or pipeline creation/dispatch will fail. + const auto* adapter = api::context()->adapter_ptr(); + const bool coopmat_runnable = + adapter->supports_cooperative_matrix() && adapter->subgroup_size() == 64; + std::vector sweep_selectors; + for (const auto& sel : coopmat_sweep_selectors) { + if (sel != "coopmat" || coopmat_runnable) { + sweep_selectors.push_back(sel); + } + } + + for (const auto& s : shapes) { + bool is_batched = s.B > 0; + + MmConfig dynamic_cfg{s.B, s.M, s.K, s.N, false, false, false}; + MmConfig const_cfg{s.B, s.M, s.K, s.N, false, false, true}; + + for (auto dtype : dtypes) { + for (auto st : storage_types) { + test_cases.push_back( + create_mm_test_case(dynamic_cfg, dtype, st, utils::kWidthPacked)); + test_cases.push_back( + create_mm_test_case(const_cfg, dtype, st, utils::kWidthPacked)); + } + + // Coopmat A/B sweep: only on aligned shapes + buffer storage. + if (coopmat_shape_eligible(s)) { + for (const auto& sel : sweep_selectors) { + MmConfig dyn = dynamic_cfg; + dyn.impl_selector = sel; + test_cases.push_back(create_mm_test_case( + dyn, dtype, utils::kBuffer, utils::kWidthPacked)); + MmConfig con = const_cfg; + con.impl_selector = sel; + test_cases.push_back(create_mm_test_case( + con, dtype, utils::kBuffer, utils::kWidthPacked)); + } + } + + if (!is_batched) { + MmConfig addmm_cfg{s.B, s.M, s.K, s.N, true, false, false}; + MmConfig addmm_const_cfg{s.B, s.M, s.K, s.N, true, false, true}; + MmConfig linear_cfg{s.B, s.M, s.K, s.N, false, true, false}; + MmConfig linear_bias_cfg{s.B, s.M, s.K, s.N, true, true, false}; + + for (auto st : storage_types) { + test_cases.push_back( + create_mm_test_case(addmm_cfg, dtype, st, utils::kWidthPacked)); + test_cases.push_back(create_mm_test_case( + addmm_const_cfg, dtype, st, utils::kWidthPacked)); + test_cases.push_back( + create_mm_test_case(linear_cfg, dtype, st, utils::kWidthPacked)); + test_cases.push_back(create_mm_test_case( + linear_bias_cfg, dtype, st, utils::kWidthPacked)); + } + + // Coopmat A/B sweep on linear paths too (only aligned shapes). + if (coopmat_shape_eligible(s)) { + for (const auto& sel : sweep_selectors) { + MmConfig lin = linear_cfg; + lin.impl_selector = sel; + test_cases.push_back(create_mm_test_case( + lin, dtype, utils::kBuffer, utils::kWidthPacked)); + MmConfig lin_bias = linear_bias_cfg; + lin_bias.impl_selector = sel; + test_cases.push_back(create_mm_test_case( + lin_bias, dtype, utils::kBuffer, utils::kWidthPacked)); + } + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("mm", "correctness") { + const std::vector shapes = { + // Accuracy shapes (float) + {0, 64, 128, 64}, + {0, 128, 256, 128}, + {0, 32, 64, 256}, + {1, 64, 128, 64}, + {1, 4, 32, 16}, + // Non-multiple-of-4 accuracy shapes (exercises scalar shader fallback) + {0, 57, 131, 43}, + {0, 33, 67, 91}, + {1, 19, 53, 37}, + {0, 64, 128, 47}, // only N unaligned + {0, 64, 47, 128}, // only K unaligned + }; + const std::vector dtypes = {vkapi::kFloat}; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector coopmat_sweep_selectors = {"tiled", "coopmat"}; + + return { + generate_mm_test_cases( + shapes, dtypes, storage_types, coopmat_sweep_selectors), + reference_impl, + mm_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("mm", "performance") { + const std::vector shapes = { + // Performance shapes (half) + {0, 4096, 1024, 256}, + {0, 4096, 256, 128}, + {0, 4096, 128, 256}, + {1, 4096, 256, 1024}, + {1, 4096, 256, 128}, + {1, 256, 4096, 64}, + {0, 4096, 64, 128}, + }; + const std::vector dtypes = {vkapi::kFloat, vkapi::kHalf}; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector coopmat_sweep_selectors = {"tiled", "coopmat"}; + + return { + generate_mm_test_cases( + shapes, dtypes, storage_types, coopmat_sweep_selectors), + reference_impl, + mm_flop_calculator}; +} + +} // namespace mm +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_q4gsw_linear.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/q4gsw_linear.cpp similarity index 58% rename from backends/vulkan/test/custom_ops/test_q4gsw_linear.cpp rename to backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/q4gsw_linear.cpp index db188d21c30..47ab929c38c 100644 --- a/backends/vulkan/test/custom_ops/test_q4gsw_linear.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/q4gsw_linear.cpp @@ -4,45 +4,19 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include +#include + #include #include -#include +#include +#include +#include #include -#include "utils.h" - -#include - -using namespace executorch::vulkan::prototyping; - -using namespace vkcompute; -static constexpr int64_t kRefDimSizeLimit = 300; - -// Linear configuration struct -struct LinearConfig { - int64_t M; // Batch size / number of rows in input - int64_t K; // Input features / columns in input, rows in weight - int64_t N; // Output features / columns in weight - int64_t group_size; // Number of input channels per quantization group - bool has_bias = false; - std::string test_case_name = "placeholder"; - std::string op_name = "linear_dq8ca_q4gsw"; -}; - -// Helper function to unpack 4-bit values from uint8 -std::pair unpack_4bit(uint8_t packed) { - // Extract lower 4 bits and upper 4 bits - int8_t lower = packed & 0x0F; - int8_t upper = (packed >> 4) & 0x0F; - - // Subtract 8 from unpacked 4-bit values - lower -= 8; - upper -= 8; - - return std::make_pair(lower, upper); -} +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q4gsw_linear { // Utility function to create a test case from a LinearConfig TestCase create_test_case_from_config( @@ -219,160 +193,264 @@ TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for quantized linear operation (for debugging) -std::vector generate_quantized_linear_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - int M = 8; - int K = 16; - int N = 16; - int group_size = 8; - - LinearConfig config = { - M, // Batch size - K, // Input features - N, // Output features - group_size, // Group size - true, // has_bias - "simple", // test_case_name - }; - - // Test with both storage types and data types for completeness - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - std::vector float_types = {vkapi::kFloat}; - - // Generate test cases for each combination - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, input_dtype)); - } +int64_t quantized_linear_flop_calculator(const TestCase& test_case) { + // Get input and weight dimensions + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); + + int64_t batch_size = input_sizes[0]; + int64_t in_features = input_sizes[1]; + int64_t out_features = output_sizes[1]; + + // Calculate FLOPs for quantized linear operation + // Each output element requires: + // - in_features multiply-accumulate operations + // - Additional operations for quantization/dequantization + int64_t output_elements = batch_size * out_features; + int64_t ops_per_output = in_features; + + // Add quantization overhead (approximate) + // - Unpack 4-bit weight: 1 op per weight element used + // - Dequantize weight: 1 op per weight element used + // - Add bias: 1 op per output element + // - For activation+weight quantized: add input quantization ops + int64_t quantization_ops = ops_per_output * 2 + 1; // Simplified estimate + + int64_t flop = output_elements * (ops_per_output + quantization_ops); + + return flop; +} + +// Create a single test case for the test_fpa_q4gsw_linear.{gemm,gemv} op. +TestCase create_test_case( + const FpaLinearConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage, + int32_t impl_selector, + bool is_gemv) { + TestCase test_case; + + const int64_t M = config.M; + const int64_t K = config.K; + const int64_t N = config.N; + const int64_t group_size = config.group_size; + + const bool is_performance = + (M > kRefDimSizeLimit || K > kRefDimSizeLimit || N > kRefDimSizeLimit); + const std::string prefix = is_performance ? "PERF" : "ACCU"; + + const std::string dtype_str = dtype_short(dtype); + const std::string shape_str = shape_bracket({M, K}) + "x[" + + std::to_string(N) + "," + std::to_string(K) + "] g" + + std::to_string(group_size); + const std::string storage_str = repr_str(storage, utils::kWidthPacked); + std::string suffix = std::string("[") + (is_gemv ? "gemv" : "gemm") + " s" + + std::to_string(impl_selector) + "]"; + suffix += config.has_bias ? " bias" : " no_bias"; + const std::string test_name = make_test_label( + prefix, dtype_str, dtype_str, shape_str, storage_str, suffix); + test_case.set_name(test_name); + + const std::string op_name = is_gemv ? "test_etvk.test_fpa_q4gsw_linear.gemv" + : "test_etvk.test_fpa_q4gsw_linear.gemm"; + test_case.set_operator_name(op_name); + + // Input: [M, K] + ValueSpec input( + {M, K}, dtype, storage, utils::kWidthPacked, DataGenType::RANDINT); + + // Weight: [N, K/2] uint8 packed 4-bit + ValueSpec weight( + {N, K / 2}, + vkapi::kByte, + storage, + utils::kWidthPacked, + DataGenType::RANDINT4); + weight.set_constant(true); + weight.set_int4(true); + + // Scales: [K/gs, N] matching input dtype (the custom op prepacks scales + // using the input tensor's dtype). + ValueSpec scales( + {K / group_size, N}, + dtype, + storage, + utils::kWidthPacked, + DataGenType::RANDOM_SCALES); + scales.set_constant(true); + + // Group size + ValueSpec gs_spec(static_cast(group_size)); + + // Bias + ValueSpec bias( + {N}, + dtype, + storage, + utils::kWidthPacked, + config.has_bias ? DataGenType::RANDOM : DataGenType::ZEROS); + bias.set_constant(true); + if (!config.has_bias) { + bias.set_none(true); } - return test_cases; + // impl_selector as int32 + ValueSpec impl_selector_spec(static_cast(impl_selector)); + + // Output: [M, N] + ValueSpec output( + {M, N}, dtype, storage, utils::kWidthPacked, DataGenType::ZEROS); + + // Tolerance: fp16 outputs use relaxed tolerance to account for f16 + // accumulation / rounding. + float base_tol = 0.05f * (static_cast(K) / 64.0f); + float tol = (dtype == vkapi::kHalf) ? (4.0f * base_tol) : base_tol; + test_case.set_abs_tolerance(tol); + + test_case.add_input_spec(input); + test_case.add_input_spec(weight); + test_case.add_input_spec(scales); + test_case.add_input_spec(gs_spec); + test_case.add_input_spec(bias); + test_case.add_input_spec(impl_selector_spec); + test_case.add_output_spec(output); + + return test_case; } -// Generate test cases for quantized linear operation -std::vector generate_quantized_linear_test_cases() { - std::vector test_cases; - - std::vector configs = { - // // Gemv test cases - // {1, 128, 64, 32}, - // {1, 256, 128, 64}, - // Gemm - {4, 64, 32, 16}, - {4, 128, 64, 32}, - {4, 256, 128, 64}, - {32, 64, 32, 16}, - {32, 128, 64, 32}, - {32, 256, 128, 64}, - {256, 128, 128, 32}, // M=256 > K=128; all dims < kRefDimSizeLimit - - // Coopmat-eligible correctness shapes (M%64==0, N%64==0, K%32==0, - // group_size%32==0). The Buffer+Half variant fires linear_q4gsw_coopmat / - // linear_dq8ca_q4gsw_coopmat and is validated against the CPU reference. - {64, 64, 64, 64}, - {64, 128, 64, 64}, - {64, 256, 128, 128}, - // With bias - {4, 64, 32, 16, true}, - {4, 128, 64, 32, true}, - {32, 128, 64, 32, true}, - // NOTE: coopmat correctness coverage is NOT in this list. The - // coopmat dispatch gate requires M%64==0, N%64==0, K%32==0; the - // smallest qualifying shape (M=64, K=64, N=64) produces enough - // cancellation outputs that fp16 accumulation drift exceeds any - // reasonable tolerance against the fp32 reference. Validating the - // coopmat shader needs a different strategy (e.g. positive-only - // inputs, or simulating fp16 accumulation in the reference). - // A couple of representative performance shapes (coopmat-eligible, - // M % 64 == 0). The full Llama 3.1 8B prefill sweep lived here during - // the study; trimmed to keep this a fast unit test. - {128, 2048, 2048, 128}, - {1024, 4096, 4096, 128}, - }; - - // Test with different storage types and data types - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - for (auto config : configs) { - std::string prefix = - (config.M < kRefDimSizeLimit && config.K < kRefDimSizeLimit && - config.N < kRefDimSizeLimit) - ? "correctness_" - : "performance_"; - std::string generated_test_case_name = prefix + std::to_string(config.M) + - "_" + std::to_string(config.K) + "_" + std::to_string(config.N) + "_g" + - std::to_string(config.group_size); - if (!config.has_bias) { - generated_test_case_name += "_no_bias"; - } +// Custom FLOP calculator: 2 * M * K * N for the linear op itself. +int64_t linear_flop_calculator(const TestCase& test_case) { + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - config.test_case_name = generated_test_case_name; - - // Iterate over both fp32 and fp16 activations so the test covers the - // _float and _half SPIR-V variants of each linear shader. Llama-on-Vulkan - // exports run with backend.vulkan.force_fp16=True, so the _half variants - // are the ones we actually hit in production. - std::vector input_dtypes = {vkapi::kFloat, vkapi::kHalf}; - - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : input_dtypes) { - // Test both activation+weight quantized and weight only quantized, but - // only if the current device supports int8 dot product - if (vkcompute::api::context() - ->adapter_ptr() - ->supports_int8_dot_product()) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, input_dtype)); - } + int64_t M = input_sizes[0]; + int64_t K = input_sizes[1]; + int64_t N = output_sizes[1]; + return 2 * M * K * N; +} - LinearConfig wo_quant_config = config; - wo_quant_config.op_name = "linear_q4gsw"; - test_cases.push_back(create_test_case_from_config( - wo_quant_config, storage_type, input_dtype)); - } - } +bool is_dq8ca(const std::string& op) { + return op.find("dq8ca") != std::string::npos; +} +bool is_4bit(const std::string& op) { + return op.find("q4gsw") != std::string::npos; +} + +// Build one test case for the given op at (storage, half dtype), no bias. +TestCase make_case(const CoopmatLinearConfig& cfg, utils::StorageType storage) { + const vkapi::ScalarType dt = vkapi::kHalf; + TestCase tc; + const std::string storage_str = + (storage == utils::kTexture3D) ? "Texture3D" : "Buffer"; + tc.set_name( + cfg.op_name + "_M" + std::to_string(cfg.M) + "_K" + + std::to_string(cfg.K) + "_N" + std::to_string(cfg.N) + "_" + storage_str); + tc.set_operator_name("et_vk." + cfg.op_name + ".default"); + + ValueSpec input( + {cfg.M, cfg.K}, dt, storage, utils::kWidthPacked, DataGenType::RANDINT); + + // dynamic per-row activation scale/zp (dq8ca only) + ValueSpec input_scale( + {1, cfg.M}, dt, storage, utils::kWidthPacked, DataGenType::RANDOM_SCALES); + input_scale.set_constant(true); + ValueSpec input_zp( + {1, cfg.M}, + vkapi::kChar, + storage, + utils::kWidthPacked, + DataGenType::RANDINT); + input_zp.set_constant(true); + + // weight + scales + sums depend on 4-bit vs 8-bit + const bool four = is_4bit(cfg.op_name); + ValueSpec qweight( + four ? std::vector{cfg.N, cfg.K / 2} + : std::vector{cfg.N, cfg.K}, + four ? vkapi::kByte : vkapi::kChar, + storage, + utils::kWidthPacked, + four ? DataGenType::RANDINT4 : DataGenType::RANDINT8); + qweight.set_constant(true); + if (four) { + qweight.set_int4(true); } - // The narrow 2-row output tile (`tiledm2`) is selected only on Mali, which - // no CI runner has, and the weight-only tests above exercise a different - // implementation. Force the variant so it is checked against the CPU - // reference everywhere. M of 2, 3 and 5 covers an exact multiple of the - // tile, one partial tile, and two tiles plus a remainder, which is where a - // wrong row bound or a missing bounds check shows up. - for (int64_t M : {2, 3, 5}) { - for (bool has_bias : {true, false}) { - LinearConfig config; - config.M = M; - config.K = 64; - config.N = 32; - config.group_size = 32; - config.has_bias = has_bias; - config.op_name = "linear_dq8ca_q4gsw"; - config.test_case_name = "correctness_tiledm2_M" + std::to_string(M) + - (has_bias ? "" : "_no_bias"); - - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : {vkapi::kFloat, vkapi::kHalf}) { - // No int8-dot-product gate here on purpose: forcing the - // fallback makes this path reachable on any GPU, which is the - // point of the variant being hard to reach otherwise. - TestCase tc = - create_test_case_from_config(config, storage_type, input_dtype); - tc.set_name(tc.name() + " [tiledm2]"); - tc.set_force_narrow_int4_tile(true); - test_cases.push_back(tc); - } - } - } + std::vector scales_size = four + ? std::vector{cfg.K / cfg.group_size, cfg.N} + : std::vector{cfg.N}; + ValueSpec weight_scales( + scales_size, + dt, + storage, + utils::kWidthPacked, + DataGenType::RANDOM_SCALES); + weight_scales.set_constant(true); + + ValueSpec weight_sums( + scales_size, + vkapi::kInt, + storage, + utils::kWidthPacked, + DataGenType::ZEROS); + weight_sums.set_constant(true); + if (four) { + compute_weight_sums_4bit_grouped( + weight_sums, qweight, cfg.K / cfg.group_size, cfg.N, cfg.group_size); + } else { + compute_weight_sums(weight_sums, qweight, cfg.N, cfg.K); } - return test_cases; + ValueSpec group_size_spec(static_cast(cfg.group_size)); + + ValueSpec bias({cfg.N}, dt, storage, utils::kWidthPacked, DataGenType::ZEROS); + bias.set_constant(true); + bias.set_none(true); + + ValueSpec output( + {cfg.M, cfg.N}, dt, storage, utils::kWidthPacked, DataGenType::ZEROS); + + // assemble per op signature + if (cfg.op_name == "linear_q4gsw") { + tc.add_input_spec(input); + tc.add_input_spec(qweight); + tc.add_input_spec(weight_scales); + tc.add_input_spec(group_size_spec); + tc.add_input_spec(bias); + } else if (cfg.op_name == "linear_dq8ca_q4gsw") { + tc.add_input_spec(input); + tc.add_input_spec(input_scale); + tc.add_input_spec(input_zp); + tc.add_input_spec(qweight); + tc.add_input_spec(weight_sums); + tc.add_input_spec(weight_scales); + tc.add_input_spec(group_size_spec); + tc.add_input_spec(bias); + } + tc.add_output_spec(output); + return tc; +} + +int64_t flop_calc(const TestCase& tc) { + const auto& in = tc.inputs()[0].get_tensor_sizes(); + const auto& out = tc.outputs()[0].get_tensor_sizes(); + const int64_t M = in[0], K = in[1], N = out[1]; + return 2 * M * N * K; // MAC = 2 flops +} + +namespace { + +// Helper function to unpack 4-bit values from uint8 +std::pair unpack_4bit(uint8_t packed) { + // Extract lower 4 bits and upper 4 bits + int8_t lower = packed & 0x0F; + int8_t upper = (packed >> 4) & 0x0F; + + // Subtract 8 from unpacked 4-bit values + lower -= 8; + upper -= 8; + + return std::make_pair(lower, upper); } // Reference implementation for 4-bit group symmetric weight quantized linear @@ -576,25 +654,16 @@ void linear_dq8ca_q4gsw_reference_impl(TestCase& test_case) { } } -// Reference for the weight-only fallback that linear_dq8ca_q4gsw takes when -// the device has no int8 dot product (see quantized_linear_impl). The op keeps -// its dq8ca signature, so the activation scale/zp and weight sums are present -// but unused: the GPU multiplies float activations by dequantized weights, and -// so does this. void linear_dq8ca_q4gsw_weight_only_reference_impl(TestCase& test_case) { const ValueSpec& input_spec = test_case.inputs()[0]; - // inputs()[1] input_scale and inputs()[2] input_zp are unused on this path const ValueSpec& weight_spec = test_case.inputs()[3]; - // inputs()[4] weight_sums is unused on this path const ValueSpec& weight_scales_spec = test_case.inputs()[5]; const ValueSpec& group_size_spec = test_case.inputs()[6]; const ValueSpec& bias_spec = test_case.inputs()[7]; - ValueSpec& output_spec = test_case.outputs()[0]; - auto input_sizes = input_spec.get_tensor_sizes(); - auto output_sizes = output_spec.get_tensor_sizes(); - + const auto input_sizes = input_spec.get_tensor_sizes(); + const auto output_sizes = output_spec.get_tensor_sizes(); const int64_t batch_size = input_sizes[0]; const int64_t in_features = input_sizes[1]; const int64_t out_features = output_sizes[1]; @@ -609,7 +678,7 @@ void linear_dq8ca_q4gsw_weight_only_reference_impl(TestCase& test_case) { throw std::invalid_argument("Unsupported dtype"); } - auto& weight_data = weight_spec.get_uint8_data(); + const auto& weight_data = weight_spec.get_uint8_data(); auto& ref_data = output_spec.get_ref_float_data(); ref_data.resize(batch_size * out_features); @@ -618,18 +687,14 @@ void linear_dq8ca_q4gsw_weight_only_reference_impl(TestCase& test_case) { float sum = 0.0f; for (int64_t in_f = 0; in_f < in_features; ++in_f) { const float input_val = input_spec.get_element(b * in_features + in_f); - const int64_t group_idx = in_f / group_size; const int64_t scales_idx = group_idx * out_features + out_f; - const int64_t weight_idx = out_f * (in_features / 2) + (in_f / 2); - const uint8_t packed_weight = weight_data[weight_idx]; - const auto unpacked = unpack_4bit(packed_weight); - const int8_t weight_4bit = + const auto unpacked = unpack_4bit(weight_data[weight_idx]); + const int8_t weight = (in_f % 2 == 0) ? unpacked.first : unpacked.second; - - const float weight_scale = weight_scales_spec.get_element(scales_idx); - sum += input_val * static_cast(weight_4bit) * weight_scale; + sum += input_val * static_cast(weight) * + weight_scales_spec.get_element(scales_idx); } if (!bias_spec.is_none()) { sum += bias_spec.get_element(out_f); @@ -639,9 +704,9 @@ void linear_dq8ca_q4gsw_weight_only_reference_impl(TestCase& test_case) { } } +} // namespace + void reference_impl(TestCase& test_case) { - // The forced-narrow-tile cases run the weight-only fallback regardless of - // what the device supports, so they need the fallback's reference. if (test_case.name().find("tiledm2") != std::string::npos) { linear_dq8ca_q4gsw_weight_only_reference_impl(test_case); } else if (test_case.operator_name().find("dq8ca") != std::string::npos) { @@ -651,56 +716,185 @@ void reference_impl(TestCase& test_case) { } } -int64_t quantized_linear_flop_calculator(const TestCase& test_case) { - // Get input and weight dimensions - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); +namespace { - int64_t batch_size = input_sizes[0]; - int64_t in_features = input_sizes[1]; - int64_t out_features = output_sizes[1]; +// Convert a ValueSpec's input data (float or half) into a flat +// std::vector for use in the reference implementation. +std::vector input_to_float_vec(const ValueSpec& spec) { + if (spec.dtype == vkapi::kFloat) { + return spec.get_float_data(); + } + if (spec.dtype == vkapi::kHalf) { + const auto& half_data = spec.get_half_data(); + std::vector out(half_data.size()); + for (size_t i = 0; i < half_data.size(); ++i) { + out[i] = half_to_float(half_data[i]); + } + return out; + } + throw std::invalid_argument( + "Reference implementation supports only float/half input dtypes."); +} - // Calculate FLOPs for quantized linear operation - // Each output element requires: - // - in_features multiply-accumulate operations - // - Additional operations for quantization/dequantization - int64_t output_elements = batch_size * out_features; - int64_t ops_per_output = in_features; +// Reference implementation: simple dequant + fp32 GEMM. Only runs for +// small shapes (gate on kRefDimSizeLimit). +void fpa_linear_q4gsw_reference_impl(TestCase& test_case) { + int32_t idx = 0; + const ValueSpec& input_spec = test_case.inputs()[idx++]; + const ValueSpec& weight_spec = test_case.inputs()[idx++]; + const ValueSpec& scales_spec = test_case.inputs()[idx++]; + const ValueSpec& gs_spec = test_case.inputs()[idx++]; + const ValueSpec& bias_spec = test_case.inputs()[idx++]; + // impl_selector is not used in the reference impl + ++idx; - // Add quantization overhead (approximate) - // - Unpack 4-bit weight: 1 op per weight element used - // - Dequantize weight: 1 op per weight element used - // - Add bias: 1 op per output element - // - For activation+weight quantized: add input quantization ops - int64_t quantization_ops = ops_per_output * 2 + 1; // Simplified estimate + ValueSpec& output_spec = test_case.outputs()[0]; - int64_t flop = output_elements * (ops_per_output + quantization_ops); + auto input_sizes = input_spec.get_tensor_sizes(); + auto output_sizes = output_spec.get_tensor_sizes(); - return flop; + int64_t M = input_sizes[0]; + int64_t K = input_sizes[1]; + int64_t N = output_sizes[1]; + int64_t group_size = gs_spec.get_int_value(); + + if (M > kRefDimSizeLimit || K > kRefDimSizeLimit || N > kRefDimSizeLimit) { + throw std::invalid_argument( + "Dimensions exceed limit for reference implementation."); + } + + std::vector input_data = input_to_float_vec(input_spec); + auto& weight_data = weight_spec.get_uint8_data(); + std::vector scales_data = input_to_float_vec(scales_spec); + std::vector bias_data; + if (!bias_spec.is_none()) { + bias_data = input_to_float_vec(bias_spec); + } + + int64_t num_output_elements = M * N; + auto& ref_data = output_spec.get_ref_float_data(); + ref_data.resize(num_output_elements); + + for (int64_t m = 0; m < M; ++m) { + for (int64_t n = 0; n < N; ++n) { + float sum = 0.0f; + for (int64_t k = 0; k < K; ++k) { + float input_val = input_data[m * K + k]; + + int64_t weight_idx = n * (K / 2) + (k / 2); + uint8_t packed = weight_data[weight_idx]; + int8_t nibble = (k % 2 == 0) + ? static_cast(packed & 0x0F) - 8 + : static_cast((packed >> 4) & 0x0F) - 8; + + int64_t group_idx = k / group_size; + float scale = scales_data[group_idx * N + n]; + + sum += input_val * static_cast(nibble) * scale; + } + if (!bias_spec.is_none()) { + sum += bias_data[n]; + } + ref_data[m * N + n] = sum; + } + } +} + +} // namespace + +void fpa_reference_impl(TestCase& test_case) { + fpa_linear_q4gsw_reference_impl(test_case); } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "4-bit Group Symmetric Weight Quantized Linear Operation Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - // Execute easy test cases using the new framework with custom FLOP calculator - auto results = execute_test_cases( - generate_quantized_linear_test_cases, - quantized_linear_flop_calculator, - "QuantizedLinearQ4GSW", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; +namespace { + +// ---- correctness reference for all four ops; oversized shapes (the perf +// cases) throw -> framework marks them SKIPPED. For dq8ca the activation +// quant round-trip (round(x/scale)+zp) is mirrored in fp32; this is exact +// (not just close) for the coopmat_correctness data, which uses scale=1/16, +// zp=0 and activations that are multiples of 1/16, so fp16-vs-fp32 +// divergence cannot occur. ---- +std::vector as_f(const ValueSpec& s) { + if (s.dtype == vkapi::kFloat) { + return s.get_float_data(); + } + const auto& h = s.get_half_data(); + std::vector o(h.size()); + for (size_t i = 0; i < h.size(); ++i) { + o[i] = half_to_float(h[i]); + } + return o; +} + +} // namespace + +void bench_reference(TestCase& tc) { + const std::string op = tc.operator_name(); + const bool dq8ca = op.find("dq8ca") != std::string::npos; + const bool four = op.find("q4gsw") != std::string::npos; + const ValueSpec& in = tc.inputs()[0]; + ValueSpec& out = tc.outputs()[0]; + const auto is = in.get_tensor_sizes(); + const int64_t M = is[0], K = is[1]; + const int64_t N = out.get_tensor_sizes()[1]; + if (M > 256 || K > 256 || N > 256) { + throw std::invalid_argument("ref: too big"); + } + // input layouts: weight-only = {in, w, w_scales, [group], bias}; + // dq8ca = {in, in_scale, in_zp, w, w_sums, w_scales, [group], bias} + const ValueSpec& w = tc.inputs()[dq8ca ? 3 : 1]; + const ValueSpec& sc = tc.inputs()[dq8ca ? 5 : 2]; + const int64_t group = four ? tc.inputs()[dq8ca ? 6 : 3].get_int_value() : K; + const ValueSpec& bias = tc.inputs()[dq8ca ? (four ? 7 : 6) : (four ? 4 : 3)]; + const bool has_bias = !bias.is_none(); + + const std::vector inf = as_f(in); + const std::vector scf = as_f(sc); + const std::vector bf = has_bias ? as_f(bias) : std::vector(); + const std::vector in_scale = + dq8ca ? as_f(tc.inputs()[1]) : std::vector(); + const std::vector& in_zp = + dq8ca ? tc.inputs()[2].get_int8_data() : std::vector(); + const std::vector& w4 = + four ? w.get_uint8_data() : std::vector(); // [N, K/2] nibbles + const std::vector& w8 = + four ? std::vector() : w.get_int8_data(); // [N, K] + + auto& ref = out.get_ref_float_data(); + ref.resize(M * N); + for (int64_t m = 0; m < M; ++m) { + const float s_in = dq8ca ? in_scale[m] : 1.0f; + const int zp = dq8ca ? int(in_zp[m]) : 0; + for (int64_t n = 0; n < N; ++n) { + float acc = 0.0f; + for (int64_t k = 0; k < K; ++k) { + float a = inf[m * K + k]; + if (dq8ca) { + float q = std::round(a / s_in) + float(zp); + q = std::min(std::max(q, -128.0f), 127.0f); + a = q - float(zp); + } + int wv; + if (four) { + const uint8_t byte = w4[n * (K / 2) + k / 2]; + const int nib = (k & 1) ? ((byte >> 4) & 0xF) : (byte & 0xF); + wv = nib - 8; + } else { + wv = w8[n * K + k]; + } + const float w_scale = four ? scf[(k / group) * N + n] : scf[n]; + acc += a * float(wv) * w_scale; + } + float r = dq8ca ? acc * s_in : acc; + if (has_bias) { + r += bf[n]; + } + ref[m * N + n] = r; + } + } } + +} // namespace q4gsw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/q4gsw_linear.h b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/q4gsw_linear.h new file mode 100644 index 00000000000..5ee95032801 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/q4gsw_linear.h @@ -0,0 +1,130 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q4gsw_linear { + +constexpr int64_t kRefDimSizeLimit = 300; + +// Linear configuration struct +struct LinearConfig { + int64_t M; // Batch size / number of rows in input + int64_t K; // Input features / columns in input, rows in weight + int64_t N; // Output features / columns in weight + int64_t group_size; // Number of input channels per quantization group + bool has_bias = false; + std::string test_case_name = "placeholder"; + std::string op_name = "linear_dq8ca_q4gsw"; +}; + +TestCase create_test_case_from_config( + const LinearConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype); + +void reference_impl(TestCase& test_case); + +int64_t quantized_linear_flop_calculator(const TestCase& test_case); + +// FPA Q4GSW Linear A/B benchmark binary. +// +// Each generated test case has an `impl_selector` arg routed to the test +// op `test_etvk.test_fpa_q4gsw_linear.{gemm,gemv}` in TestFpaQ4gswLinear.cpp: +// +// GEMM (is_gemv=false): +// 0 -> PROD (et_vk.q4gsw_linear.default; dtype-based +// picker) 1 -> GEMM_W_4X8 (forced non-tin GEMM, nc buffer +// weight) 2 -> GEMM_TIN_W_4X8 (forced tin GEMM, nc buffer +// weight) 3 -> LEGACY (et_vk.linear_q4gsw.default +// legacy shaders) +// +// GEMV (is_gemv=true): +// 0 -> PROD (et_vk.q4gsw_linear.default; +// dtype-based picker) 1 -> GEMV_W_4X8 (forced gemv with +// subgroup broadcast) 2 -> GEMV_W_4X8_NOSG (forced gemv +// without subgroup broadcast) 3 -> LEGACY (et_vk.linear_q4gsw.default +// legacy shaders) 13 -> GEMV_COOP_W_4X8_NC_BUFFER (coop GEMV reusing the +// production +// nc-buffer prepack — same payload as +// W_4X8 GEMM/TIN GEMM/sg-GEMV; +// == g1w64 decomposition) +// 14 -> GEMV_COOP_..._G1W64 (force NUM_GROUPS=1, +// WORKERS_PER_GROUP=64) 15 -> GEMV_COOP_..._G4W16 (force +// NUM_GROUPS=4, WORKERS_PER_GROUP=16) 16 -> GEMV_COOP_..._G8W8 (force +// NUM_GROUPS=8, WORKERS_PER_GROUP=8) +// +// Selectors 14-16 pin the coop nc-buffer GEMV to an explicit reduction +// decomposition regardless of N. The production picker (pick_coop_variant_for_N +// in Q4gswLinear.cpp) only chooses g4w16 / g8w8 at PERF-sized N where the +// reference impl is skipped; these forced selectors give g4w16 / g8w8 numeric +// (ACCU) coverage at small N. Production picker behavior is unchanged. +// +// Selector 3 (LEGACY) is the in-prod q4gsw linear path. It uses a different +// prepack (pack_q4_linear_weight) and shader family +// (linear_q4gsw_tiled_* / linear_q4gsw_coop_*); the framework's per-shader +// timing breakdown will pick those up automatically. + +// Linear configuration struct. +struct FpaLinearConfig { + int64_t M; + int64_t K; + int64_t N; + int64_t group_size; + bool has_bias = false; +}; + +TestCase create_test_case( + const FpaLinearConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage, + int32_t impl_selector, + bool is_gemv); + +void fpa_reference_impl(TestCase& test_case); + +int64_t linear_flop_calculator(const TestCase& test_case); + +// Consolidated coopmat-vs-tiled microbenchmark for the two int4-quantized +// linear types at Llama 3.1 8B prefill shapes: +// 4w = linear_q4gsw (weight-only int4) +// 8da4w = linear_dq8ca_q4gsw (dyn-act int8 x int4 weight) +// +// Baseline (tiled) is selected by Texture3D+Half output storage; coopmat is +// selected by Buffer+Half (the runtime gate in QuantizedLinear.cpp picks the +// _coopmat shader when M%64==0, N%64==0, K%32==0, subgroup==64). The CPU +// reference only runs for the small shapes in coopmat_correctness. + +struct CoopmatLinearConfig { + int64_t M; + int64_t K; + int64_t N; + int64_t group_size; // only meaningful for 4-bit + std::string op_name; +}; + +bool is_dq8ca(const std::string& op); + +bool is_4bit(const std::string& op); + +TestCase make_case(const CoopmatLinearConfig& cfg, utils::StorageType storage); + +void bench_reference(TestCase& tc); + +int64_t flop_calc(const TestCase& tc); + +} // namespace q4gsw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/coopmat_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/coopmat_sets.cpp new file mode 100644 index 00000000000..826bdde55c5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/coopmat_sets.cpp @@ -0,0 +1,154 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q4gsw_linear { + +namespace { + +// POSITIVE well-conditioned data (no fp16 cancellation): activations are +// multiples of 1/16 in [0.5,1.375]; int4 nibbles in {9..14} (-> weight +1..+6) +// / int8 weights in {1..6}. For dq8ca the per-row activation scale is forced to +// 1/16 with zp=0 so the dynamic int8 quant round-trip is EXACT in both fp16 and +// fp32 (quantized values 8..22) and the fp32 reference is valid. fp16~=fp32 +// throughout, so a tight tolerance validates shader structure (catches +// zero-subtile bugs) while ignoring benign fp16 noise. +void set_well_conditioned_data(TestCase& t, const CoopmatLinearConfig& cfg) { + const bool dq = is_dq8ca(cfg.op_name); + const bool four = is_4bit(cfg.op_name); + auto& hin = t.inputs()[0].get_half_data(); + for (size_t i = 0; i < hin.size(); ++i) { + hin[i] = float_to_half(0.5f + 0.125f * float(i % 8)); + } + const size_t w_idx = dq ? 3 : 1; + if (four) { + auto& wq = t.inputs()[w_idx].get_uint8_data(); + const std::array kPos = {0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE}; + for (size_t i = 0; i < wq.size(); ++i) { + wq[i] = kPos[i % 6]; + } + } else { + auto& wq = t.inputs()[w_idx].get_int8_data(); + for (size_t i = 0; i < wq.size(); ++i) { + wq[i] = int8_t(1 + (i % 6)); + } + } + if (dq) { + auto& hs = t.inputs()[1].get_half_data(); + std::fill(hs.begin(), hs.end(), float_to_half(0.0625f)); + auto& zp = t.inputs()[2].get_int8_data(); + std::fill(zp.begin(), zp.end(), int8_t(0)); + // weights were overwritten above -> recompute the sums + if (four) { + compute_weight_sums_4bit_grouped( + t.inputs()[4], + t.inputs()[w_idx], + cfg.K / cfg.group_size, + cfg.N, + cfg.group_size); + } else { + compute_weight_sums(t.inputs()[4], t.inputs()[w_idx], cfg.N, cfg.K); + } + } + t.set_abs_tolerance(0.5f); + t.set_rel_tolerance(0.05f); +} + +std::vector generate_coopmat_test_cases( + const std::vector& shapes, + const std::vector& ops, + const std::vector& storage_types, + bool well_conditioned_data) { + std::vector cases; + for (const auto& op : ops) { + for (const auto& shape : shapes) { + CoopmatLinearConfig cfg{shape.M, shape.K, shape.N, shape.group_size, op}; + for (auto st : storage_types) { + TestCase t = make_case(cfg, st); + if (well_conditioned_data) { + set_well_conditioned_data(t, cfg); + } + cases.push_back(std::move(t)); + } + } + } + return cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q4gsw_linear", "coopmat_correctness") { + // Correctness: small aligned {64,128,64} cases for ALL FOUR ops; the buffer + // case fires the coopmat shader, validated by bench_reference (the + // performance cases are skipped by it). Texture3D = tiled, Buffer = coopmat. + // Shapes align to BOTH coopmat geometries (64x64x32 legacy, 128x128x16 + // double-buffered); the second shape dispatches a multi-workgroup grid for + // both, covering the gl_WorkGroupID-derived tile offsets in the store + // address math. + return { + generate_coopmat_test_cases( + { + {64, 128, 64, 64, ""}, + {128, 256, 128, 64, ""}, + {128, 128, 128, 64, ""}, + {256, 256, 256, 64, ""}, + // Discriminators for the tiled-texture cube-shape failure: + {128, 128, 256, 64, ""}, // M == K only + {256, 128, 128, 64, ""}, // K == N only + {64, 128, 256, 64, ""}, // K > M, K < N + {256, 128, 64, 64, ""}, // K < M, K > N + }, + /*ops=*/{"linear_q4gsw", "linear_dq8ca_q4gsw"}, + /*storage_types=*/{utils::kTexture3D, utils::kBuffer}, + /*well_conditioned_data=*/true), + bench_reference, + flop_calc, + /*warmup_runs=*/3, + /*benchmark_runs=*/5}; +} + +namespace { + +constexpr int64_t kM = 1024; +constexpr int64_t kGroup = 128; + +} // namespace + +REGISTER_TEST_CASE_SET("q4gsw_linear", "coopmat_performance") { + return { + generate_coopmat_test_cases( + { + // Llama 3.1 8B linear weight shapes (K,N) at prefill M (multiple + // of 64 so coopmat fires). + {kM, 4096, 4096, kGroup, ""}, // q_proj / o_proj + {kM, 4096, 1024, kGroup, ""}, // k_proj / v_proj (GQA) + {kM, 4096, 14336, kGroup, ""}, // gate_proj / up_proj + {kM, 14336, 4096, kGroup, ""}, // down_proj + }, + /*ops=*/{"linear_q4gsw", "linear_dq8ca_q4gsw"}, + // Texture3D selects the tiled shader, Buffer the coopmat shader. + /*storage_types=*/{utils::kTexture3D, utils::kBuffer}, + /*well_conditioned_data=*/false), + bench_reference, + flop_calc, + /*warmup_runs=*/3, + /*benchmark_runs=*/5}; +} + +} // namespace q4gsw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/debug_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/debug_set.cpp new file mode 100644 index 00000000000..64e1d6999c7 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/debug_set.cpp @@ -0,0 +1,58 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q4gsw_linear { + +namespace { + +std::vector generate_debug_test_cases( + const LinearConfig& config, + const std::vector& storage_types, + const std::vector& input_dtypes) { + std::vector test_cases; + + for (const auto& storage_type : storage_types) { + for (const auto& input_dtype : input_dtypes) { + test_cases.push_back( + create_test_case_from_config(config, storage_type, input_dtype)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q4gsw_linear", "debug") { + // Single simple configuration for debugging + return { + generate_debug_test_cases( + { + 8, // Batch size + 16, // Input features + 16, // Output features + 8, // Group size + true, // has_bias + "simple", // test_case_name + "linear_dq8ca_q4gsw", // op_name + }, + /*storage_types=*/{utils::kTexture3D, utils::kBuffer}, + /*input_dtypes=*/{vkapi::kFloat}), + reference_impl, + quantized_linear_flop_calculator}; +} + +} // namespace q4gsw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/fpa_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/fpa_sets.cpp new file mode 100644 index 00000000000..9e7902a6008 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/fpa_sets.cpp @@ -0,0 +1,264 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q4gsw_linear { + +namespace { + +// One case per M x (K, N) x dtype x storage x impl_selector, in that nesting +// order. +struct FpaSweep { + std::vector Ms; + std::vector> nk_shapes; // (K, N) + std::vector dtypes; + std::vector storages; + std::vector impl_selectors; + bool is_gemv; +}; + +std::vector generate_fpa_test_cases( + int64_t group_size, + const std::vector& sweeps) { + std::vector test_cases; + for (const auto& sweep : sweeps) { + for (int64_t M : sweep.Ms) { + for (const auto& shape : sweep.nk_shapes) { + const int64_t K = shape.first; + const int64_t N = shape.second; + FpaLinearConfig cfg{M, K, N, group_size}; + for (auto dtype : sweep.dtypes) { + for (auto storage : sweep.storages) { + for (int32_t selector : sweep.impl_selectors) { + test_cases.push_back(create_test_case( + cfg, dtype, storage, selector, sweep.is_gemv)); + } + } + } + } + } + } + return test_cases; +} + +const std::vector kDtypes = {vkapi::kFloat, vkapi::kHalf}; +const std::vector kStorages = { + utils::kBuffer, + utils::kTexture3D}; + +// ACCU correctness shapes (under kRefDimSizeLimit=300). Exercise selectors +// PROD (0) and forced nosg (2). Forced sg (1) is intentionally skipped: +// sg requires subgroupSize==64 and produces incorrect results on Mali +// (subgroupSize==16); on those devices the PROD picker correctly routes +// to the nosg variant. N must be a multiple of 128 (= 2 * LWG.x) so the +// GEMV shader has no early-exit threads in any workgroup. +const std::vector> kAccuShapes = { + // (K, N) + {128, 128}, + {256, 256}, +}; +// Selector 13 (nc Buffer, reuses production prepack) included for ACCU +// coverage of the coop nc weight-binding variant. +const std::vector kAccuSelectors = {0, 2, 13}; + +// Forced coop-reduction-decomposition ACCU coverage (selectors 14/15/16 = +// g1w64 / g4w16 / g8w8). The production picker (pick_coop_variant_for_N) +// only selects g4w16 (10244096) at PERF-sized N, where +// every dim exceeds kRefDimSizeLimit=300 so the reference impl is skipped and +// those reduction decompositions get zero numeric validation. These cases +// pin each decomposition regardless of N at small shapes (all dims <= 300) +// so the reference runs and proves g4w16 / g8w8 compute the SAME result as +// the reference — the M=1 decode path actually shipped for Qwen3 / Llama. +// +// Each WG of variant gN produces N*8 outputs (g1w64 -> 8, g4w16 -> 32, +// g8w8 -> 64), so the N values below tile cleanly into all three: N=128 +// (16 / 4 / 2 WGs) and N=256 (32 / 8 / 4 WGs). The shader also handles a +// ragged final WG, but clean tiles keep the test intent unambiguous. K is +// swept over {64, 128, 256} (all multiples of group_size=32 and <= 300) so +// the K-loop reduction is exercised across short and longer accumulations. +const std::vector> kCoopForcedAccuShapes = { + // (K, N) — all dims <= 300 so linear_q4gsw_reference_impl runs. + {64, 128}, + {128, 256}, + {256, 128}, +}; +// 14 -> g1w64, 15 -> g4w16, 16 -> g8w8. g1w64 included for symmetry. +const std::vector kCoopForcedSelectors = {14, 15, 16}; + +// Non-aligned-N coverage for the W_4X8 GEMM path. The fp32 GEMM issues a +// 16B ivec4 weight load that spans two consecutive (k4, n4) ivec2 tiles +// along N, so N4 must be even (== N a multiple of 8) at the buffer-stride +// level. The prepack pads the weight buffer's row stride to next-even N4 +// and fills the OOB tiles with bias-zero nibbles; these accuracy cases +// exercise that padding path on shapes with N % 8 != 0. Only the new W_4X8 +// family is tested (selectors 0, 1, 2, 5, 6) — selector 3 (LEGACY) uses a +// different prepack and supports arbitrary N. +const std::vector> kNonAlignedNkShapes = { + // (K, N) — K kept under kRefDimSizeLimit so reference impl runs. + {128, 12}, + {128, 20}, +}; +const std::vector kNonAlignedSelectors = {0, 1, 2}; + +} // namespace + +REGISTER_TEST_CASE_SET("q4gsw_linear", "fpa_correctness") { + return { + generate_fpa_test_cases( + /*group_size=*/32, + { + // GEMV + {{1}, + kAccuShapes, + kDtypes, + kStorages, + kAccuSelectors, + /*is_gemv=*/true}, + {{1}, + kCoopForcedAccuShapes, + kDtypes, + kStorages, + kCoopForcedSelectors, + /*is_gemv=*/true}, + // GEMM + {{32}, + kNonAlignedNkShapes, + kDtypes, + kStorages, + kNonAlignedSelectors, + /*is_gemv=*/false}, + // Small ACCU shape (M=32, K=128, N=128) under kRefDimSizeLimit so + // the reference impl runs. Sanity-checks GEMM correctness during + // iteration. + {{32}, + {{128, 128}}, + kDtypes, + {utils::kTexture3D}, + /*impl_selectors=*/{0}, + /*is_gemv=*/false}, + // M-tail ACCU shapes. These exercise final partial GEMM tiles for + // both the fp32 direct-input path (tile height 4) and the fp16 + // TIN path (tile height 8). + {{31, 33}, + {{128, 128}}, + kDtypes, + {utils::kTexture3D}, + /*impl_selectors=*/{0}, + /*is_gemv=*/false}, + }), + fpa_reference_impl, + linear_flop_calculator, + /*warmup_runs=*/3, + /*benchmark_runs=*/10}; +} + +namespace { + +// Canonical N/K shapes for LLM hidden-size sweeps. +const std::vector> kNkShapes = { + // (K, N) + {1024, 2048}, + {4096, 4096}, + // {4096, 14336}, // Large-N case can make the full benchmark binary + // unstable. +}; + +// GEMV PERF selectors: PROD (0), forced sg (1), forced nosg (2), LEGACY (3), +// nc-Buffer coop (13, reuses production prepack — single-format +// prefill+decode). +const std::vector kGemvPerfSelectors = {0, 1, 2, 3, 13}; + +// LLM-decode-shape PERF cells (M=1 GEMV, group_size=32). Mirrors the actual +// per-layer linear shapes seen during decode profiling on Llama 3.2 1B and +// Qwen3 0.6B; the original Phase 2 corpus (1024/2048/4096 x 2048/4096/11008) +// under-samples these and missed the regression where sg-GEMV (selector 1) +// is 15-22% slower per dispatch than LEGACY coop (selector 3) on Adreno 750. +// +// All N values here are multiples of 128 (= 2 * LWG.x for the GEMV shader), +// so the GEMV shader has no early-exit threads. N=512 is the Llama 3.2 1B +// k_proj/v_proj projection (GQA) and is a multiple of 4 (the prepack +// requirement: prepack_q4_w_4x8_nc_buffer enforces N % 4 == 0). +// +// Default storage is fp16 + Tex3D - that's the actual decode config and the +// shape combo where the regression was observed. We additionally exercise +// K=2048,N=2048 under fp32 + Tex3D and fp32 + Buffer to confirm the +// regression isn't fp16-Tex3D-specific. All four selectors (PROD, sg, nosg, +// LEGACY) are exercised. +const std::vector> kLlmGemvShapes = { + // (K, N) - Llama 3.2 1B + {2048, 512}, // k_proj / v_proj (GQA) + {2048, 2048}, // q_proj + {2048, 8192}, // gate_proj / up_proj + {8192, 2048}, // down_proj + // (K, N) - Qwen3 0.6B + {1024, 1024}, // k_proj / v_proj + {1024, 2048}, // q_proj (also overlaps with original corpus) + {1024, 3072}, // gate_proj / up_proj + {3072, 1024}, // down_proj +}; + +// Selectors exercised in the GEMM PERF/ACCU sweep: PROD (0), forced non-tin +// / tin GEMM (1, 2), legacy (3). +// +// Selector 3 is the legacy in-prod q4gsw linear path +// (et_vk.linear_q4gsw.default) registered in QuantizedLinear.cpp. +const std::vector kGemmSelectors = {0, 1, 2, 3}; + +} // namespace + +REGISTER_TEST_CASE_SET("q4gsw_linear", "fpa_performance") { + return { + generate_fpa_test_cases( + /*group_size=*/32, + { + // GEMV sweep: M = 1 x N/K shapes x dtype x storage x + // impl_selector. + {{1}, + kNkShapes, + kDtypes, + kStorages, + kGemvPerfSelectors, + /*is_gemv=*/true}, + {{1}, + kLlmGemvShapes, + {vkapi::kHalf}, + {utils::kTexture3D}, + kGemvPerfSelectors, + /*is_gemv=*/true}, + // Diversity sanity check: K=2048,N=2048 under fp32 + {Tex3D, + // Buffer} to confirm the regression isn't fp16-Tex3D-specific. + {{1}, + {{2048, 2048}}, + {vkapi::kFloat}, + {utils::kTexture3D, utils::kBuffer}, + kGemvPerfSelectors, + /*is_gemv=*/true}, + // GEMM sweep: M in {32, 128, 256} x N/K shapes x dtype x storage + // x impl_selector. + {{32, 128, 256}, + kNkShapes, + kDtypes, + kStorages, + kGemmSelectors, + /*is_gemv=*/false}, + }), + fpa_reference_impl, + linear_flop_calculator, + /*warmup_runs=*/3, + /*benchmark_runs=*/10}; +} + +} // namespace q4gsw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/q4gsw_linear_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/q4gsw_linear_sets.cpp new file mode 100644 index 00000000000..be0e57dbe42 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q4gsw_linear/test_sets/q4gsw_linear_sets.cpp @@ -0,0 +1,164 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q4gsw_linear { + +namespace { + +std::vector generate_test_cases( + const std::vector& configs, + const std::vector& storage_types, + const std::vector& input_dtypes, + const std::vector& ops, + bool include_narrow_tile_cases = false) { + std::vector test_cases; + + const bool supports_int8_dot_product = + vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product(); + + for (auto config : configs) { + std::string prefix = + (config.M < kRefDimSizeLimit && config.K < kRefDimSizeLimit && + config.N < kRefDimSizeLimit) + ? "correctness_" + : "performance_"; + std::string generated_test_case_name = prefix + std::to_string(config.M) + + "_" + std::to_string(config.K) + "_" + std::to_string(config.N) + "_g" + + std::to_string(config.group_size); + if (!config.has_bias) { + generated_test_case_name += "_no_bias"; + } + + config.test_case_name = generated_test_case_name; + + for (const auto& storage_type : storage_types) { + for (const auto& input_dtype : input_dtypes) { + for (const auto& op : ops) { + if (is_dq8ca(op) && !supports_int8_dot_product) { + continue; + } + LinearConfig op_config = config; + op_config.op_name = op; + test_cases.push_back(create_test_case_from_config( + op_config, storage_type, input_dtype)); + } + } + } + } + + if (include_narrow_tile_cases) { + for (int64_t M : {2, 3, 5}) { + for (bool has_bias : {true, false}) { + LinearConfig config; + config.M = M; + config.K = 64; + config.N = 32; + config.group_size = 32; + config.has_bias = has_bias; + config.op_name = "linear_dq8ca_q4gsw"; + config.test_case_name = "correctness_tiledm2_M" + std::to_string(M) + + (has_bias ? "" : "_no_bias"); + + for (const auto& storage_type : storage_types) { + for (const auto& input_dtype : input_dtypes) { + TestCase test_case = + create_test_case_from_config(config, storage_type, input_dtype); + test_case.set_name(test_case.name() + " [tiledm2]"); + test_case.set_force_narrow_int4_tile(true); + test_cases.push_back(test_case); + } + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q4gsw_linear", "correctness") { + return { + generate_test_cases( + { + // // Gemv test cases + // {1, 128, 64, 32}, + // {1, 256, 128, 64}, + // Gemm + {4, 64, 32, 16}, + {4, 128, 64, 32}, + {4, 256, 128, 64}, + {32, 64, 32, 16}, + {32, 128, 64, 32}, + {32, 256, 128, 64}, + {256, 128, 128, 32}, // M=256 > K=128; all dims < kRefDimSizeLimit + + // Coopmat-eligible correctness shapes (M%64==0, N%64==0, + // K%32==0, group_size%32==0). The Buffer+Half variant fires + // linear_q4gsw_coopmat / linear_dq8ca_q4gsw_coopmat and is + // validated against the CPU reference. + {64, 64, 64, 64}, + {64, 128, 64, 64}, + {64, 256, 128, 128}, + // With bias + {4, 64, 32, 16, true}, + {4, 128, 64, 32, true}, + {32, 128, 64, 32, true}, + // NOTE: coopmat correctness coverage is NOT in this list. The + // coopmat dispatch gate requires M%64==0, N%64==0, K%32==0; the + // smallest qualifying shape (M=64, K=64, N=64) produces enough + // cancellation outputs that fp16 accumulation drift exceeds any + // reasonable tolerance against the fp32 reference. Validating + // the coopmat shader needs a different strategy (e.g. + // positive-only inputs, or simulating fp16 accumulation in the + // reference). + }, + /*storage_types=*/{utils::kTexture3D, utils::kBuffer}, + // Both fp32 and fp16 activations, so the test covers the _float and + // _half SPIR-V variants of each linear shader. Llama-on-Vulkan + // exports run with backend.vulkan.force_fp16=True, so the _half + // variants are the ones we actually hit in production. + /*input_dtypes=*/{vkapi::kFloat, vkapi::kHalf}, + // Activation+weight quantized and weight-only quantized. + /*ops=*/{"linear_dq8ca_q4gsw", "linear_q4gsw"}, + /*include_narrow_tile_cases=*/true), + reference_impl, + quantized_linear_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q4gsw_linear", "performance") { + return { + generate_test_cases( + { + // A couple of representative performance shapes + // (coopmat-eligible, M % 64 == 0). The full Llama 3.1 8B prefill + // sweep lived here during the study; trimmed to keep this a fast + // unit test. + {128, 2048, 2048, 128}, + {1024, 4096, 4096, 128}, + }, + /*storage_types=*/{utils::kTexture3D, utils::kBuffer}, + // fp16 is the production path (backend.vulkan.force_fp16=True); fp32 + // covers the _float shader variants. + /*input_dtypes=*/{vkapi::kFloat, vkapi::kHalf}, + // Activation+weight quantized and weight-only quantized. + /*ops=*/{"linear_dq8ca_q4gsw", "linear_q4gsw"}), + reference_impl, + quantized_linear_flop_calculator}; +} + +} // namespace q4gsw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_q8csw_conv2d.cpp b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/q8csw_conv2d.cpp similarity index 75% rename from backends/vulkan/test/custom_ops/test_q8csw_conv2d.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/q8csw_conv2d.cpp index be7a4cab2b8..8af65739d8e 100644 --- a/backends/vulkan/test/custom_ops/test_q8csw_conv2d.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/q8csw_conv2d.cpp @@ -4,20 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include -#include -#include -#include "conv2d_utils.h" -#include "utils.h" - -#include - -using namespace executorch::vulkan::prototyping; +#include -using namespace vkcompute; +#include +#include +#include +#include +#include -static constexpr int64_t kRefDimSizeLimit = 100; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_conv2d { // Utility function to create a test case from a Conv2dConfig TestCase create_test_case_from_config( @@ -178,196 +176,40 @@ TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for quantized conv2d operation (for debugging) -std::vector generate_quantized_conv2d_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - Conv2dConfig config = { - OutInChannels(8, 3), // channels (out, in) - InputSize2D(8, 8), // input_size (h, w) - KernelSize(3, 3), // kernel - Stride(1, 1), // stride - Padding(0, 0), // padding - Dilation(1, 1), // dilation - 1, // groups - }; - - // Test with both storage types and data types for completeness - std::vector storage_types = {utils::kTexture3D}; - std::vector float_types = {vkapi::kFloat}; - - // Generate test cases for each combination - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, input_dtype)); - } +// Custom FLOP calculator for quantized conv2d operation +int64_t quantized_conv2d_flop_calculator(const TestCase& test_case) { + int kernel_idx = 4; + if (test_case.operator_name().find("q8ta") != std::string::npos) { + kernel_idx = 7; } + // Get input and weight dimensions + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - return test_cases; -} + const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); -// Generate test cases for quantized conv2d operation -std::vector generate_quantized_conv2d_test_cases() { - std::vector test_cases; - - std::vector configs = { - {OutInChannels(32, 3), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(32, 16), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - // One output channel case - {OutInChannels(1, 32), - InputSize2D(55, 55), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - - // Stride 2 convolutions - {OutInChannels(32, 3), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - // Different kernel sizes - {OutInChannels(32, 16), - InputSize2D(28, 28), - KernelSize(5, 5), - Stride(1, 1), - Padding(2, 2), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(14, 14), - KernelSize(7, 7), - Stride(1, 1), - Padding(3, 3), - Dilation(1, 1), - 1}, - - // Dilated convolutions - {OutInChannels(32, 16), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(1, 1), - Padding(2, 2), - Dilation(2, 2), - 1}, - {OutInChannels(64, 32), - InputSize2D(16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(3, 3), - Dilation(3, 3), - 1}, - - // Grouped convolutions - {OutInChannels(32, 32), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 4}, - {OutInChannels(64, 64), - InputSize2D(16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 8}, - // Performance test cases - {OutInChannels(256, 128), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(128, 64), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - {OutInChannels(128, 1024), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}}; - - // Test with different storage types and data types - std::vector storage_types = {utils::kTexture3D}; - - // Generate test cases for each combination - for (auto& config : configs) { - for (const auto& storage_type : storage_types) { - // Generate test case name programmatically - bool is_performance = config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit || - config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - std::string prefix = is_performance ? "performance_" : "correctness_"; - std::string suffix = std::to_string(config.channels.out) + "/" + - std::to_string(config.channels.in) + "_" + - std::to_string(config.input_size.h) + "/" + - std::to_string(config.input_size.w) + "_" + - std::to_string(config.kernel.h) + "/" + - std::to_string(config.kernel.w); - - config.op_name = "conv2d_q8ta_q8csw"; - config.test_case_name = prefix + suffix; - // The default operator tested is activation + weight quantized conv2d; - // however, only test this if the int8 dot product extension is supported - if (vkcompute::api::context() - ->adapter_ptr() - ->supports_int8_dot_product()) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, vkapi::kFloat)); - } + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t C_out = output_sizes[1]; + int64_t K_h = kernel_sizes[0]; + int64_t K_w = kernel_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; - Conv2dConfig wo_quant_config = config; - wo_quant_config.op_name = "conv2d_q8csw"; - test_cases.push_back(create_test_case_from_config( - wo_quant_config, storage_type, vkapi::kFloat)); - } - } + // Calculate FLOPs for quantized conv2d operation + // Each output element requires: + // - C_in * K_h * K_w multiply-accumulate operations + // - Additional operations for quantization/dequantization + int64_t output_elements = N * C_out * H_out * W_out; + int64_t ops_per_output = C_in * K_h * K_w; + + int64_t flop = output_elements * (ops_per_output); - return test_cases; + return flop; } +namespace { + // Reference implementation for weight only quantized conv2d (fp accumulation) void conv2d_q8csw_reference_impl(TestCase& test_case) { int32_t idx = 0; @@ -444,7 +286,8 @@ void conv2d_q8csw_reference_impl(TestCase& test_case) { auto& ref_data = output_spec.get_ref_float_data(); ref_data.resize(num_output_elements); - const int in_features = utils::align_up_4(C_in_per_group * K_h * K_w); + const int in_features = + utils::safe_downcast(utils::align_up_4(C_in_per_group * K_h * K_w)); // Perform weight-only quantized conv2d operation (fp accumulation) for (int64_t n = 0; n < N; ++n) { @@ -584,7 +427,8 @@ void conv2d_q8ta_q8csw_reference_impl(TestCase& test_case) { auto& ref_data = output_spec.get_ref_float_data(); ref_data.resize(num_output_elements); - const int in_features = utils::align_up_4(C_in_per_group * K_h * K_w); + const int in_features = + utils::safe_downcast(utils::align_up_4(C_in_per_group * K_h * K_w)); // Perform activation and weight quantized conv2d operation (int accumulation) for (int64_t n = 0; n < N; ++n) { @@ -674,6 +518,8 @@ void conv2d_q8ta_q8csw_reference_impl(TestCase& test_case) { } } +} // namespace + void reference_impl(TestCase& test_case) { if (test_case.operator_name().find("q8ta") != std::string::npos) { conv2d_q8ta_q8csw_reference_impl(test_case); @@ -682,58 +528,7 @@ void reference_impl(TestCase& test_case) { } } -// Custom FLOP calculator for quantized conv2d operation -int64_t quantized_conv2d_flop_calculator(const TestCase& test_case) { - int kernel_idx = 4; - if (test_case.operator_name().find("q8ta") != std::string::npos) { - kernel_idx = 7; - } - // Get input and weight dimensions - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - - const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t C_out = output_sizes[1]; - int64_t K_h = kernel_sizes[0]; - int64_t K_w = kernel_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - // Calculate FLOPs for quantized conv2d operation - // Each output element requires: - // - C_in * K_h * K_w multiply-accumulate operations - // - Additional operations for quantization/dequantization - int64_t output_elements = N * C_out * H_out * W_out; - int64_t ops_per_output = C_in * K_h * K_w; - - int64_t flop = output_elements * (ops_per_output); - - return flop; -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(true); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Quantized Conv2d Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - // Execute test cases using the new framework with custom FLOP calculator - auto results = execute_test_cases( - generate_quantized_conv2d_test_cases, - quantized_conv2d_flop_calculator, - "QuantizedConv2d", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8csw_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/q8csw_conv2d.h b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/q8csw_conv2d.h new file mode 100644 index 00000000000..0cf3f5a4278 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/q8csw_conv2d.h @@ -0,0 +1,32 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_conv2d { + +constexpr int64_t kRefDimSizeLimit = 100; + +TestCase create_test_case_from_config( + const Conv2dConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype); + +void reference_impl(TestCase& test_case); + +int64_t quantized_conv2d_flop_calculator(const TestCase& test_case); + +} // namespace q8csw_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/test_sets/q8csw_conv2d_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/test_sets/q8csw_conv2d_sets.cpp new file mode 100644 index 00000000000..6bee244b04d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_conv2d/test_sets/q8csw_conv2d_sets.cpp @@ -0,0 +1,202 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_conv2d { + +namespace { + +std::vector generate_test_cases( + const std::vector& configs, + const std::vector& storage_types) { + std::vector test_cases; + + // Generate test cases for each combination + for (auto config : configs) { + for (const auto& storage_type : storage_types) { + // Generate test case name programmatically + bool is_performance = config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit || + config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + std::string prefix = is_performance ? "performance_" : "correctness_"; + std::string suffix = std::to_string(config.channels.out) + "/" + + std::to_string(config.channels.in) + "_" + + std::to_string(config.input_size.h) + "/" + + std::to_string(config.input_size.w) + "_" + + std::to_string(config.kernel.h) + "/" + + std::to_string(config.kernel.w); + + config.op_name = "conv2d_q8ta_q8csw"; + config.test_case_name = prefix + suffix; + // The default operator tested is activation + weight quantized conv2d; + // however, only test this if the int8 dot product extension is supported + if (vkcompute::api::context() + ->adapter_ptr() + ->supports_int8_dot_product()) { + test_cases.push_back( + create_test_case_from_config(config, storage_type, vkapi::kFloat)); + } + + Conv2dConfig wo_quant_config = config; + wo_quant_config.op_name = "conv2d_q8csw"; + test_cases.push_back(create_test_case_from_config( + wo_quant_config, storage_type, vkapi::kFloat)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8csw_conv2d", "correctness") { + const std::vector configs = { + {OutInChannels(32, 3), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(32, 16), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + // One output channel case + {OutInChannels(1, 32), + InputSize2D(55, 55), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + + // Stride 2 convolutions + {OutInChannels(32, 3), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + // Different kernel sizes + {OutInChannels(32, 16), + InputSize2D(28, 28), + KernelSize(5, 5), + Stride(1, 1), + Padding(2, 2), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(14, 14), + KernelSize(7, 7), + Stride(1, 1), + Padding(3, 3), + Dilation(1, 1), + 1}, + + // Dilated convolutions + {OutInChannels(32, 16), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(1, 1), + Padding(2, 2), + Dilation(2, 2), + 1}, + {OutInChannels(64, 32), + InputSize2D(16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(3, 3), + Dilation(3, 3), + 1}, + + // Grouped convolutions + {OutInChannels(32, 32), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 4}, + {OutInChannels(64, 64), + InputSize2D(16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 8}, + }; + const std::vector storage_types = {utils::kTexture3D}; + + return { + generate_test_cases(configs, storage_types), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8csw_conv2d", "performance") { + const std::vector configs = { + // Performance test cases + {OutInChannels(256, 128), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(128, 64), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(128, 1024), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + }; + const std::vector storage_types = {utils::kTexture3D}; + + return { + generate_test_cases(configs, storage_types), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8csw_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_q8csw_linear.cpp b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/q8csw_linear.cpp similarity index 76% rename from backends/vulkan/test/custom_ops/test_q8csw_linear.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8csw_linear/q8csw_linear.cpp index 5e9154b5648..4718ab4310e 100644 --- a/backends/vulkan/test/custom_ops/test_q8csw_linear.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/q8csw_linear.cpp @@ -4,29 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include -#include -#include -#include "utils.h" - -#include - -using namespace executorch::vulkan::prototyping; +#include -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 300; +#include +#include +#include +#include +#include -// Linear configuration struct -struct LinearConfig { - int64_t M; // Batch size / number of rows in input - int64_t K; // Input features / columns in input, rows in weight - int64_t N; // Output features / columns in weight - bool has_bias = true; - std::string test_case_name = "placeholder"; - std::string op_name = "linear_q8ta_q8csw"; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_linear { // Utility function to create a test case from a LinearConfig TestCase create_test_case_from_config( @@ -158,95 +147,42 @@ TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for quantized linear operation (for debugging) -std::vector generate_quantized_linear_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - int M = 4; - int K = 4; - int N = 4; - - LinearConfig config = { - M, // Batch size - K, // Input features - N, // Output features - true, // has_bias - "simple", // test_case_name - }; - - // Test with both storage types and data types for completeness - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - std::vector float_types = {vkapi::kFloat}; - - // Generate test cases for each combination - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - test_cases.push_back( - create_test_case_from_config(config, storage_type, input_dtype)); - } +int64_t quantized_linear_flop_calculator(const TestCase& test_case) { + int input_idx = 0; + int weight_idx = 1; + if (test_case.operator_name().find("q8ta") != std::string::npos) { + input_idx = 0; + weight_idx = 3; } - return test_cases; -} + // Get input and weight dimensions + const auto& input_sizes = test_case.inputs()[input_idx].get_tensor_sizes(); + const auto& weight_sizes = test_case.inputs()[weight_idx].get_tensor_sizes(); -// Generate test cases for quantized linear operation -std::vector generate_quantized_linear_test_cases() { - std::vector test_cases; - - std::vector configs = { - {4, 64, 32}, - {4, 128, 64}, - {4, 256, 128}, - {32, 64, 32}, - {32, 128, 64}, - {32, 256, 128}, - // No bias tests - {32, 128, 64, false}, - {32, 256, 128, false}, - {256, 2048, 2048}, - {512, 2048, 2048}, - {1024, 2048, 2048}, - }; - - // Test with different storage types and data types - std::vector storage_types = { - utils::kTexture3D, utils::kBuffer}; - - for (auto config : configs) { - std::string prefix = - (config.M < kRefDimSizeLimit && config.K < kRefDimSizeLimit && - config.N < kRefDimSizeLimit) - ? "correctness_" - : "performance_"; - std::string generated_test_case_name = prefix + std::to_string(config.M) + - "_" + std::to_string(config.K) + "_" + std::to_string(config.N); - if (!config.has_bias) { - generated_test_case_name += "_no_bias"; - } + int64_t batch_size = input_sizes[0]; + int64_t in_features = input_sizes[1]; + int64_t out_features = weight_sizes[0]; - config.test_case_name = generated_test_case_name; + // Calculate FLOPs for quantized linear operation + // Each output element requires: + // - in_features multiply-accumulate operations + // - Additional operations for quantization/dequantization + int64_t output_elements = batch_size * out_features; + int64_t ops_per_output = in_features; - for (const auto& storage_type : storage_types) { - if (vkcompute::api::context() - ->adapter_ptr() - ->supports_int8_dot_product()) { - // Test both activation+weight quantized and weight only quantized - test_cases.push_back( - create_test_case_from_config(config, storage_type, vkapi::kFloat)); - } + // Add quantization overhead (approximate) + // - Dequantize input: 1 op per input element used + // - Dequantize weight: 1 op per weight element used + // - Add bias: 1 op per output element + int64_t quantization_ops = ops_per_output + 1; // Simplified estimate - LinearConfig wo_quant_config = config; - wo_quant_config.op_name = "linear_q8csw"; - test_cases.push_back(create_test_case_from_config( - wo_quant_config, storage_type, vkapi::kFloat)); - } - } + int64_t flop = output_elements * (ops_per_output + quantization_ops); - return test_cases; + return flop; } +namespace { + // Reference implementation for weight only quantized linear void linear_q8csw_reference_impl(TestCase& test_case) { int32_t idx = 0; @@ -425,6 +361,8 @@ void linear_q8ta_q8csw_reference_impl(TestCase& test_case) { } } +} // namespace + void reference_impl(TestCase& test_case) { if (test_case.operator_name().find("q8ta") != std::string::npos) { linear_q8ta_q8csw_reference_impl(test_case); @@ -433,59 +371,7 @@ void reference_impl(TestCase& test_case) { } } -int64_t quantized_linear_flop_calculator(const TestCase& test_case) { - int input_idx = 0; - int weight_idx = 1; - if (test_case.operator_name().find("q8ta") != std::string::npos) { - input_idx = 0; - weight_idx = 3; - } - - // Get input and weight dimensions - const auto& input_sizes = test_case.inputs()[input_idx].get_tensor_sizes(); - const auto& weight_sizes = test_case.inputs()[weight_idx].get_tensor_sizes(); - - int64_t batch_size = input_sizes[0]; - int64_t in_features = input_sizes[1]; - int64_t out_features = weight_sizes[0]; - - // Calculate FLOPs for quantized linear operation - // Each output element requires: - // - in_features multiply-accumulate operations - // - Additional operations for quantization/dequantization - int64_t output_elements = batch_size * out_features; - int64_t ops_per_output = in_features; - - // Add quantization overhead (approximate) - // - Dequantize input: 1 op per input element used - // - Dequantize weight: 1 op per weight element used - // - Add bias: 1 op per output element - int64_t quantization_ops = ops_per_output + 1; // Simplified estimate - - int64_t flop = output_elements * (ops_per_output + quantization_ops); - - return flop; -} - -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Quantized Linear Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_quantized_linear_test_cases, - quantized_linear_flop_calculator, - "QuantizedLinear", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8csw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/q8csw_linear.h b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/q8csw_linear.h new file mode 100644 index 00000000000..3a3df55ad63 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/q8csw_linear.h @@ -0,0 +1,43 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_linear { + +constexpr int64_t kRefDimSizeLimit = 300; + +// Linear configuration struct +struct LinearConfig { + int64_t M; // Batch size / number of rows in input + int64_t K; // Input features / columns in input, rows in weight + int64_t N; // Output features / columns in weight + bool has_bias = true; + std::string test_case_name = "placeholder"; + std::string op_name = "linear_q8ta_q8csw"; +}; + +TestCase create_test_case_from_config( + const LinearConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype); + +void reference_impl(TestCase& test_case); + +int64_t quantized_linear_flop_calculator(const TestCase& test_case); + +} // namespace q8csw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/debug_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/debug_set.cpp new file mode 100644 index 00000000000..5402016b048 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/debug_set.cpp @@ -0,0 +1,56 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_linear { + +namespace { + +std::vector generate_single_op_test_cases( + const std::vector& configs, + const std::vector& storage_types, + const std::vector& dtypes) { + std::vector test_cases; + + for (const auto& config : configs) { + for (const auto& storage_type : storage_types) { + for (const auto& input_dtype : dtypes) { + test_cases.push_back( + create_test_case_from_config(config, storage_type, input_dtype)); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8csw_linear", "debug") { + const std::vector configs = { + // Single simple configuration for debugging + {4, 4, 4, true, "simple"}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + const std::vector dtypes = {vkapi::kFloat}; + + return { + generate_single_op_test_cases(configs, storage_types, dtypes), + reference_impl, + quantized_linear_flop_calculator}; +} + +} // namespace q8csw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/q8csw_linear_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/q8csw_linear_sets.cpp new file mode 100644 index 00000000000..4cbea88deb9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8csw_linear/test_sets/q8csw_linear_sets.cpp @@ -0,0 +1,98 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8csw_linear { + +namespace { + +std::vector generate_test_cases( + const std::vector& configs, + const std::vector& storage_types) { + std::vector test_cases; + + for (auto config : configs) { + std::string prefix = + (config.M < kRefDimSizeLimit && config.K < kRefDimSizeLimit && + config.N < kRefDimSizeLimit) + ? "correctness_" + : "performance_"; + std::string generated_test_case_name = prefix + std::to_string(config.M) + + "_" + std::to_string(config.K) + "_" + std::to_string(config.N); + if (!config.has_bias) { + generated_test_case_name += "_no_bias"; + } + + config.test_case_name = generated_test_case_name; + + for (const auto& storage_type : storage_types) { + if (vkcompute::api::context() + ->adapter_ptr() + ->supports_int8_dot_product()) { + // Test both activation+weight quantized and weight only quantized + test_cases.push_back( + create_test_case_from_config(config, storage_type, vkapi::kFloat)); + } + + LinearConfig wo_quant_config = config; + wo_quant_config.op_name = "linear_q8csw"; + test_cases.push_back(create_test_case_from_config( + wo_quant_config, storage_type, vkapi::kFloat)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8csw_linear", "correctness") { + const std::vector configs = { + {4, 64, 32}, + {4, 128, 64}, + {4, 256, 128}, + {32, 64, 32}, + {32, 128, 64}, + {32, 256, 128}, + // No bias tests + {32, 128, 64, false}, + {32, 256, 128, false}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + + return { + generate_test_cases(configs, storage_types), + reference_impl, + quantized_linear_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8csw_linear", "performance") { + const std::vector configs = { + {256, 2048, 2048}, + {512, 2048, 2048}, + {1024, 2048, 2048}, + }; + const std::vector storage_types = { + utils::kTexture3D, utils::kBuffer}; + + return { + generate_test_cases(configs, storage_types), + reference_impl, + quantized_linear_flop_calculator}; +} + +} // namespace q8csw_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_q8ta_binary.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/binary.cpp similarity index 60% rename from backends/vulkan/test/custom_ops/test_q8ta_binary.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/binary/binary.cpp index 4a2654b00db..7c878a2b485 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_binary.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/binary.cpp @@ -4,28 +4,19 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include +#include + #include #include -#include -#include -#include -#include "utils.h" - -using namespace executorch::vulkan::prototyping; +#include +#include -static constexpr int64_t kRefDimSizeLimit = 512; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_binary { -// Configuration struct for q8ta binary testing -struct Q8taBinaryConfig { - std::vector shape; // Tensor shape (can be any dimensionality) - std::string test_case_name = "placeholder"; - std::string op_name = "q8ta_add"; - // Shape of input B; empty means same as `shape`. Inputs are broadcast - // against each other. - std::vector other_shape = {}; -}; +namespace { std::vector broadcast_sizes( const std::vector& a, @@ -39,8 +30,6 @@ std::vector broadcast_sizes( return out; } -// Maps a contiguous index into a tensor of out_sizes to the contiguous index -// of the element it reads from a tensor of `sizes` broadcast to out_sizes. int64_t broadcast_src_idx( int64_t out_idx, const std::vector& out_sizes, @@ -62,6 +51,8 @@ int64_t broadcast_src_idx( return src_idx; } +} // namespace + // Utility function to create a test case from a Q8taBinaryConfig TestCase create_test_case_from_config( const Q8taBinaryConfig& config, @@ -69,7 +60,7 @@ TestCase create_test_case_from_config( vkapi::ScalarType input_dtype, utils::GPUMemoryLayout fp_memory_layout, utils::GPUMemoryLayout quant_layout, - bool const_b = false) { + bool const_b) { TestCase test_case; const std::vector& other_shape = @@ -180,161 +171,6 @@ TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for q8ta_add operation (for debugging) -std::vector generate_q8ta_add_easy_cases() { - std::vector test_cases; - - std::vector> shapes = { - {1, 16, 16, 16}, // 4D: [N, C, H, W] - {1, 144}, // 2D: exercises block config with ndim < 4 - {1, 90}, // 2D: matches skin_seg model's keypoint/bbox tensor sizes - }; - - // Quantized memory layouts to test - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - for (const auto& shape : shapes) { - Q8taBinaryConfig config = {shape, "ACCU"}; - for (const auto& quant_layout : quant_layouts) { - test_cases.push_back(create_test_case_from_config( - config, - /*storage_type=*/utils::kBuffer, - /*input_dtype=*/vkapi::kFloat, - /*fp_memory_layout=*/utils::kWidthPacked, - quant_layout)); - test_cases.push_back(create_test_case_from_config( - config, - /*fp_storage_type=*/utils::kBuffer, - /*input_dtype=*/vkapi::kFloat, - /*fp_layout=*/utils::kWidthPacked, - quant_layout, - /*const_b=*/true)); - } - } - - return test_cases; -} - -// Generate test cases for q8ta_add operation -std::vector generate_q8ta_add_test_cases() { - std::vector test_cases; - - // Shapes to test - std::vector> shapes = { - // 1D tensors - {144}, - {90}, - - // 3D tensors - {1, 16, 32}, - {1, 3, 64}, - - // 2D tensors (exercises block config with ndim < 4) - {1, 144}, - {1, 90}, - {1, 4}, - {3, 32}, - - // Small test cases for correctness - {1, 3, 16, 16}, - {1, 8, 32, 32}, - {1, 16, 24, 24}, - {1, 32, 12, 12}, - {1, 1, 64, 64}, - {1, 3, 64, 64}, - {1, 4, 16, 16}, - - // Different tensor sizes - {1, 8, 20, 20}, - {1, 16, 14, 14}, - {1, 8, 28, 28}, - - // Odd tensor sizes - {1, 3, 15, 15}, - {1, 13, 31, 31}, - {1, 17, 23, 23}, - - // Performance test cases (larger tensors) - {1, 64, 128, 128}, - {1, 32, 64, 64}, - {1, 128, 56, 56}, - {1, 128, 128, 128}, - }; - - // Quantized memory layouts to test - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - std::vector configs; - for (const auto& shape : shapes) { - // Generate test case name prefix from shape dimensions - std::string prefix = "ACCU"; - for (const auto& dim : shape) { - if (dim > kRefDimSizeLimit) { - prefix = "PERF"; - break; - } - } - Q8taBinaryConfig config; - config.shape = shape; - config.test_case_name = prefix; - configs.push_back(config); - } - - // Broadcast cases: {input A shape, input B shape} - std::vector, std::vector>> - broadcast_shapes = { - // Per-video context row added to every frame row - {{1, 60, 512}, {1, 1, 512}}, - {{1, 1, 512}, {1, 60, 512}}, - {{1, 16, 32}, {1, 16, 1}}, - {{1, 16, 32}, {32}}, - {{1, 8, 16, 16}, {1, 8, 1, 1}}, - {{1, 8, 16, 16}, {1, 1, 16, 16}}, - {{2, 8, 6, 6}, {1, 8, 6, 6}}, - {{1, 13, 7, 9}, {1, 1, 7, 1}}, - }; - for (const auto& [shape, other_shape] : broadcast_shapes) { - Q8taBinaryConfig config; - config.shape = shape; - config.test_case_name = "ACCU"; - config.other_shape = other_shape; - configs.push_back(config); - } - - // Generate all combinations - for (const auto& config : configs) { - for (const auto& quant_layout : quant_layouts) { - test_cases.push_back(create_test_case_from_config( - config, - /*storage_type=*/utils::kBuffer, - /*input_dtype=*/vkapi::kFloat, - /*fp_memory_layout=*/utils::kWidthPacked, - quant_layout)); - test_cases.push_back(create_test_case_from_config( - config, - /*fp_storage_type=*/utils::kBuffer, - /*fp_input_dtype=*/vkapi::kFloat, - /*fp_layout=*/utils::kWidthPacked, - quant_layout, - /*const_b=*/true)); - } - } - - return test_cases; -} - // Reference implementation for quantized add operation void q8ta_add_reference_impl(TestCase& test_case) { int32_t idx = 0; @@ -440,32 +276,7 @@ void q8ta_add_reference_impl(TestCase& test_case) { } } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(false); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Q8TA Binary Add Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = q8ta_add_reference_impl; - - auto results = execute_test_cases( -#ifdef DEBUG_MODE - generate_q8ta_add_easy_cases, -#else - generate_q8ta_add_test_cases, -#endif - "Q8taBinaryAdd", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_binary +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/binary.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/binary.h new file mode 100644 index 00000000000..5e5541397bf --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/binary.h @@ -0,0 +1,45 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_binary { + +constexpr int64_t kRefDimSizeLimit = 512; + +// Configuration struct for q8ta binary testing +struct Q8taBinaryConfig { + std::vector shape; // Tensor shape (can be any dimensionality) + std::string test_case_name = "placeholder"; + std::string op_name = "q8ta_add"; + // Shape of input B; empty means same as `shape`. Inputs are broadcast + // against each other. + std::vector other_shape = {}; +}; + +TestCase create_test_case_from_config( + const Q8taBinaryConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype, + utils::GPUMemoryLayout fp_memory_layout, + utils::GPUMemoryLayout quant_layout, + bool const_b = false); + +void q8ta_add_reference_impl(TestCase& test_case); + +} // namespace q8ta_binary +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/test_sets/binary_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/test_sets/binary_sets.cpp new file mode 100644 index 00000000000..6470f4f7373 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/binary/test_sets/binary_sets.cpp @@ -0,0 +1,168 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_binary { + +namespace { + +std::vector generate_test_cases( + const std::vector>& shapes, + const std::vector, std::vector>>& + broadcast_shapes, + const std::vector& quant_layouts) { + std::vector test_cases; + + std::vector configs; + for (const auto& shape : shapes) { + // Generate test case name prefix from shape dimensions + std::string prefix = "ACCU"; + for (const auto& dim : shape) { + if (dim > kRefDimSizeLimit) { + prefix = "PERF"; + break; + } + } + + Q8taBinaryConfig config; + config.shape = shape; + config.test_case_name = prefix; + configs.push_back(config); + } + for (const auto& [shape, other_shape] : broadcast_shapes) { + Q8taBinaryConfig config; + config.shape = shape; + config.test_case_name = "ACCU"; + config.other_shape = other_shape; + configs.push_back(config); + } + + // Generate all combinations + for (const auto& config : configs) { + for (const auto& quant_layout : quant_layouts) { + test_cases.push_back(create_test_case_from_config( + config, + /*storage_type=*/utils::kBuffer, + /*input_dtype=*/vkapi::kFloat, + /*fp_memory_layout=*/utils::kWidthPacked, + quant_layout)); + test_cases.push_back(create_test_case_from_config( + config, + /*storage_type=*/utils::kBuffer, + /*input_dtype=*/vkapi::kFloat, + /*fp_memory_layout=*/utils::kWidthPacked, + quant_layout, + /*const_b=*/true)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/binary", "correctness") { + // Shapes to test + const std::vector> shapes = { + // 1D tensors + {144}, + {90}, + + // 3D tensors + {1, 16, 32}, + {1, 3, 64}, + + // 2D tensors (exercises block config with ndim < 4) + {1, 144}, + {1, 90}, + {1, 4}, + {3, 32}, + + // Small test cases for correctness + {1, 3, 16, 16}, + {1, 8, 32, 32}, + {1, 16, 24, 24}, + {1, 32, 12, 12}, + {1, 1, 64, 64}, + {1, 3, 64, 64}, + {1, 4, 16, 16}, + + // Different tensor sizes + {1, 8, 20, 20}, + {1, 16, 14, 14}, + {1, 8, 28, 28}, + + // Odd tensor sizes + {1, 3, 15, 15}, + {1, 13, 31, 31}, + {1, 17, 23, 23}, + + // Performance test cases (larger tensors) + {1, 64, 128, 128}, + {1, 32, 64, 64}, + {1, 128, 56, 56}, + {1, 128, 128, 128}, + }; + // Quantized memory layouts to test + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + // Broadcast cases: {input A shape, input B shape} + const std::vector, std::vector>> + broadcast_shapes = { + // Per-video context row added to every frame row + {{1, 60, 512}, {1, 1, 512}}, + {{1, 1, 512}, {1, 60, 512}}, + {{1, 16, 32}, {1, 16, 1}}, + {{1, 16, 32}, {32}}, + {{1, 8, 16, 16}, {1, 8, 1, 1}}, + {{1, 8, 16, 16}, {1, 1, 16, 16}}, + {{2, 8, 6, 6}, {1, 8, 6, 6}}, + {{1, 13, 7, 9}, {1, 1, 7, 1}}, + }; + return { + generate_test_cases(shapes, broadcast_shapes, quant_layouts), + q8ta_add_reference_impl}; +} + +// Easy test cases for q8ta_add operation (for debugging) +REGISTER_TEST_CASE_SET("q8ta/binary", "debug") { + const std::vector> shapes = { + {1, 16, 16, 16}, // 4D: [N, C, H, W] + {1, 144}, // 2D: exercises block config with ndim < 4 + {1, 90}, // 2D: matches skin_seg model's keypoint/bbox tensor sizes + }; + // Quantized memory layouts to test + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + const std::vector, std::vector>> + broadcast_shapes = {}; + return { + generate_test_cases(shapes, broadcast_shapes, quant_layouts), + q8ta_add_reference_impl}; +} + +} // namespace q8ta_binary +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_q8ta_clone.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/clone.cpp similarity index 52% rename from backends/vulkan/test/custom_ops/test_q8ta_clone.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/clone/clone.cpp index 4936c3702cf..7ed1363695c 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_clone.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/clone.cpp @@ -4,29 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include +#include + #include #include -#include +#include +#include #include -#include "utils.h" - -#include - -// #define DEBUG_MODE -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 512; - -// Configuration struct for q8ta clone testing -struct Q8taCloneConfig { - std::vector shape; // Tensor shape (can be any dimensionality) - std::string test_case_name = "placeholder"; - std::string op_name = "q8ta_clone_test"; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_clone { // Utility function to create a test case from a Q8taCloneConfig TestCase create_test_case_from_config( @@ -109,140 +98,6 @@ TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for q8ta_clone operation (for debugging) -std::vector generate_q8ta_clone_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - Q8taCloneConfig config = { - {1, 16, 16, 16}, // shape: [N, C, H, W] - "ACCU", // test_case_name - }; - - // FP memory layouts to test - std::vector fp_layouts = { - utils::kWidthPacked, - utils::kChannelsPacked, - }; - - // Quantized memory layouts to test - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - std::vector storage_types = {utils::kBuffer}; - std::vector float_types = {vkapi::kFloat}; - - // Generate test cases for each combination (same layout for input and output) - for (const auto& fp_layout : fp_layouts) { - for (const auto& quant_layout : quant_layouts) { - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - // Same layout: should be a simple copy - test_cases.push_back(create_test_case_from_config( - config, - storage_type, - input_dtype, - fp_layout, - quant_layout, - quant_layout)); - } - } - } - } - - return test_cases; -} - -// Generate test cases for q8ta_clone operation -std::vector generate_q8ta_clone_test_cases() { - std::vector test_cases; - - // Shapes to test - std::vector> shapes = { - // Small test cases for correctness - {1, 3, 16, 16}, - {1, 8, 32, 32}, - {1, 16, 24, 24}, - {1, 32, 12, 12}, - {1, 1, 64, 64}, - {1, 3, 64, 64}, - {1, 4, 16, 16}, - - // Different tensor sizes - {1, 8, 20, 20}, - {1, 16, 14, 14}, - {1, 8, 28, 28}, - - // Odd tensor sizes - {1, 3, 15, 15}, - {1, 13, 31, 31}, - {1, 17, 23, 23}, - - // Performance test cases (larger tensors) - {1, 64, 128, 128}, - {1, 32, 64, 64}, - {1, 128, 56, 56}, - {1, 128, 128, 128}, - }; - - // FP memory layouts to test - std::vector fp_layouts = { - utils::kWidthPacked, - utils::kChannelsPacked, - }; - - // Quantized memory layouts to test - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - // Test with buffer storage only - std::vector storage_types = {utils::kBuffer}; - - // Generate all combinations - for (const auto& shape : shapes) { - // Generate test case name prefix from shape dimensions - std::string prefix = "ACCU"; - for (const auto& dim : shape) { - if (dim > kRefDimSizeLimit) { - prefix = "PERF"; - break; - } - } - - for (const auto& fp_layout : fp_layouts) { - for (const auto& inp_quant_layout : quant_layouts) { - for (const auto& outp_quant_layout : quant_layouts) { - for (const auto& storage_type : storage_types) { - Q8taCloneConfig config; - config.shape = shape; - config.test_case_name = prefix; - - test_cases.push_back(create_test_case_from_config( - config, - storage_type, - vkapi::kFloat, - fp_layout, - inp_quant_layout, - outp_quant_layout)); - } - } - } - } - } - - return test_cases; -} - // Reference implementation for q8ta_clone operation // Since clone just copies data, the result should be the same as // quantize-dequantize @@ -315,32 +170,18 @@ void q8ta_clone_reference_impl(TestCase& test_case) { } } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(false); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Q8TA Clone Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = q8ta_clone_reference_impl; - - auto results = execute_test_cases( -#ifdef DEBUG_MODE - generate_q8ta_clone_easy_cases, -#else - generate_q8ta_clone_test_cases, -#endif - "Q8taClone", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; +std::string get_prefix(const std::vector& shape) { + std::string prefix = "ACCU"; + for (const auto& dim : shape) { + if (dim > kRefDimSizeLimit) { + prefix = "PERF"; + break; + } + } + return prefix; } + +} // namespace q8ta_clone +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/clone.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/clone.h new file mode 100644 index 00000000000..2e0677fd646 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/clone.h @@ -0,0 +1,45 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_clone { + +constexpr int64_t kRefDimSizeLimit = 512; + +// Configuration struct for q8ta clone testing +struct Q8taCloneConfig { + std::vector shape; // Tensor shape (can be any dimensionality) + std::string test_case_name = "placeholder"; + std::string op_name = "q8ta_clone_test"; +}; + +TestCase create_test_case_from_config( + const Q8taCloneConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype, + utils::GPUMemoryLayout fp_memory_layout, + utils::GPUMemoryLayout inp_quant_layout, + utils::GPUMemoryLayout outp_quant_layout); + +void q8ta_clone_reference_impl(TestCase& test_case); + +// "PERF" if any dim of shape exceeds kRefDimSizeLimit, else "ACCU". +std::string get_prefix(const std::vector& shape); + +} // namespace q8ta_clone +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/correctness_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/correctness_set.cpp new file mode 100644 index 00000000000..509f249672b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/correctness_set.cpp @@ -0,0 +1,107 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_clone { + +namespace { + +std::vector generate_test_cases( + const std::vector>& shapes, + const std::vector& fp_layouts, + const std::vector& quant_layouts, + const std::vector& storage_types) { + std::vector test_cases; + + // Generate all combinations + for (const auto& shape : shapes) { + std::string prefix = get_prefix(shape); + + for (const auto& fp_layout : fp_layouts) { + for (const auto& inp_quant_layout : quant_layouts) { + for (const auto& outp_quant_layout : quant_layouts) { + for (const auto& storage_type : storage_types) { + Q8taCloneConfig config; + config.shape = shape; + config.test_case_name = prefix; + + test_cases.push_back(create_test_case_from_config( + config, + storage_type, + vkapi::kFloat, + fp_layout, + inp_quant_layout, + outp_quant_layout)); + } + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/clone", "correctness") { + // Shapes to test + const std::vector> shapes = { + // Small test cases for correctness + {1, 3, 16, 16}, + {1, 8, 32, 32}, + {1, 16, 24, 24}, + {1, 32, 12, 12}, + {1, 1, 64, 64}, + {1, 3, 64, 64}, + {1, 4, 16, 16}, + + // Different tensor sizes + {1, 8, 20, 20}, + {1, 16, 14, 14}, + {1, 8, 28, 28}, + + // Odd tensor sizes + {1, 3, 15, 15}, + {1, 13, 31, 31}, + {1, 17, 23, 23}, + + // Performance test cases (larger tensors) + {1, 64, 128, 128}, + {1, 32, 64, 64}, + {1, 128, 56, 56}, + {1, 128, 128, 128}, + }; + // FP memory layouts to test + const std::vector fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + // Quantized memory layouts to test + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + // Test with buffer storage only + const std::vector storage_types = {utils::kBuffer}; + return { + generate_test_cases(shapes, fp_layouts, quant_layouts, storage_types), + q8ta_clone_reference_impl}; +} + +} // namespace q8ta_clone +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/debug_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/debug_set.cpp new file mode 100644 index 00000000000..97f869ab02e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/clone/test_sets/debug_set.cpp @@ -0,0 +1,84 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_clone { + +namespace { + +std::vector generate_same_layout_test_cases( + const std::vector>& shapes, + const std::vector& fp_layouts, + const std::vector& quant_layouts, + const std::vector& storage_types, + const std::vector& float_types) { + std::vector test_cases; + + for (const auto& shape : shapes) { + Q8taCloneConfig config = {shape, get_prefix(shape)}; + + // Generate test cases for each combination (same layout for input and + // output) + for (const auto& fp_layout : fp_layouts) { + for (const auto& quant_layout : quant_layouts) { + for (const auto& storage_type : storage_types) { + for (const auto& input_dtype : float_types) { + // Same layout: should be a simple copy + test_cases.push_back(create_test_case_from_config( + config, + storage_type, + input_dtype, + fp_layout, + quant_layout, + quant_layout)); + } + } + } + } + } + + return test_cases; +} + +} // namespace + +// Easy test cases for q8ta_clone operation (for debugging) +REGISTER_TEST_CASE_SET("q8ta/clone", "debug") { + // Single simple configuration for debugging + const std::vector> shapes = { + {1, 16, 16, 16}, // shape: [N, C, H, W] + }; + // FP memory layouts to test + const std::vector fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + // Quantized memory layouts to test + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + const std::vector storage_types = {utils::kBuffer}; + const std::vector float_types = {vkapi::kFloat}; + return { + generate_same_layout_test_cases( + shapes, fp_layouts, quant_layouts, storage_types, float_types), + q8ta_clone_reference_impl}; +} + +} // namespace q8ta_clone +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.cpp new file mode 100644 index 00000000000..cd80172a3d3 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.cpp @@ -0,0 +1,605 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +// Utility function to create a test case from a Conv2dConfig +TestCase create_test_case_from_config_with_layouts( + const Conv2dConfig& config, + vkapi::ScalarType input_dtype, + utils::StorageType fp_storage_type, + utils::GPUMemoryLayout input_int8_memory_layout, + utils::GPUMemoryLayout output_int8_memory_layout, + const std::string& impl_selector = "", + const Im2colUnsignedTestOptions* im2col_options = nullptr, + const float input_scale_val = 0.008123f, + const DataGenType input_data_gen = DataGenType::RANDOM) { + TestCase test_case; + + // Calculate output dimensions + int64_t H_out = config.get_output_height(); + int64_t W_out = config.get_output_width(); + + // Input tensor (float/half) - [N, C_in, H_in, W_in] + std::vector input_size = { + config.batch, + config.channels.in, + config.input_size.h, + config.input_size.w}; + + utils::GPUMemoryLayout fp_memory_layout = fp_storage_type == utils::kBuffer + ? utils::kWidthPacked + : utils::kChannelsPacked; + + // Create test case name + std::string prefix = config.test_case_name.substr(0, 4); // "ACCU" or "PERF" + std::string dtype_str = dtype_short(input_dtype); + std::string in_shape = "[" + std::to_string(config.batch) + "," + + std::to_string(config.channels.in) + "," + + std::to_string(config.input_size.h) + "," + + std::to_string(config.input_size.w) + "]"; + std::string weight_shape = "[" + std::to_string(config.channels.out) + "," + + std::to_string(config.channels.in / config.groups) + "," + + std::to_string(config.kernel.h) + "," + std::to_string(config.kernel.w) + + "]"; + std::string shape_str = in_shape + "x" + weight_shape + " s" + + std::to_string(config.stride.h) + " p" + + std::to_string(config.padding.h) + " d" + + std::to_string(config.dilation.h) + " g" + std::to_string(config.groups); + std::string storage_str = repr_str(utils::kBuffer, input_int8_memory_layout); + if (input_int8_memory_layout != output_int8_memory_layout) { + storage_str += "->" + repr_str(utils::kBuffer, output_int8_memory_layout); + } + std::string suffix = impl_selector.empty() ? "" : "[" + impl_selector + "]"; + std::string test_name = make_test_label( + prefix, dtype_str, dtype_str, shape_str, storage_str, suffix); + test_case.set_name(test_name); + + // Set the operator name for the test case - use the unified test operator + std::string operator_name = "test_etvk.test_q8ta_conv2d.default"; + test_case.set_operator_name(operator_name); + + ValueSpec input_tensor( + input_size, + input_dtype, + fp_storage_type, + fp_memory_layout, + input_data_gen); + + if (debugging()) { + print_valuespec_data(input_tensor, "input_tensor"); + } + + ValueSpec input_scale(input_scale_val); + + const int32_t input_zero_point_val = + im2col_options == nullptr ? 2 : im2col_options->input_zero_point; + ValueSpec input_zero_point(input_zero_point_val); + + if (im2col_options != nullptr && + im2col_options->use_accumulator_limit_values) { + input_tensor.ensure_data_generated(2401); + std::fill( + input_tensor.get_float_data().begin(), + input_tensor.get_float_data().end(), + (127.0f - input_zero_point_val) * input_scale_val); + } else if (im2col_options != nullptr && im2col_options->use_extreme_values) { + input_tensor.ensure_data_generated(2401); + constexpr std::array values = {-128, -1, 0, 1, 127}; + std::vector& input_data = input_tensor.get_float_data(); + for (size_t i = 0; i < input_data.size(); ++i) { + input_data.at(i) = (static_cast(values.at(i % values.size())) - + input_zero_point_val) * + input_scale_val; + } + } + + // Quantized weight tensor (int8) - [C_out, C_in_per_group * K_h * K_w] + // Memory layout: height, width, then channels - in_c is innermost (stride 1) + // in the second dimension + const int64_t in_channels_per_group = config.channels.in / config.groups; + const int64_t in_features = utils::align_up_4( + in_channels_per_group * config.kernel.h * config.kernel.w); + std::vector weight_size = {config.channels.out, in_features}; + ValueSpec quantized_weight( + weight_size, + vkapi::kChar, // int8 for quantized weights + fp_storage_type, + utils::kWidthPacked, + DataGenType::RANDINT8); + quantized_weight.set_constant(true); + + if (im2col_options != nullptr && + im2col_options->use_accumulator_limit_values) { + quantized_weight.ensure_data_generated(2402); + std::fill( + quantized_weight.get_int8_data().begin(), + quantized_weight.get_int8_data().end(), + 127); + } else if (im2col_options != nullptr && im2col_options->use_extreme_values) { + quantized_weight.ensure_data_generated(2402); + constexpr std::array values = {-128, -1, 0, 1, 127}; + std::vector& weight_data = quantized_weight.get_int8_data(); + for (size_t i = 0; i < weight_data.size(); ++i) { + weight_data.at(i) = values.at((i * 3 + 1) % values.size()); + } + } + + if (debugging()) { + print_valuespec_data(quantized_weight, "weight_tensor"); + } + + const int64_t aligned_out_channels = utils::align_up_4(config.channels.out); + + // Weight quantization scales (float/half, per-channel) + ValueSpec weight_scales( + {aligned_out_channels}, // Per output channel + input_dtype, + fp_storage_type, + utils::kWidthPacked, + DataGenType::RANDOM_SCALES); + weight_scales.set_constant(true); + if (im2col_options != nullptr) { + weight_scales.ensure_data_generated(2403); + std::fill( + weight_scales.get_float_data().begin(), + weight_scales.get_float_data().end(), + im2col_options->weight_scale); + } + + ValueSpec weight_sums( + {aligned_out_channels}, // Per output channel + vkapi::kInt, + fp_storage_type, + utils::kWidthPacked, + DataGenType::ZEROS); + weight_sums.set_constant(true); + + // Compute weight_sums data based on quantized weights + compute_weight_sums( + weight_sums, quantized_weight, config.channels.out, in_features); + + // Bias (optional, float/half) - [C_out] + ValueSpec bias( + {aligned_out_channels}, // Per output channel + input_dtype, + fp_storage_type, + utils::kWidthPacked, + DataGenType::ZEROS); + bias.set_constant(true); + if (im2col_options != nullptr && !im2col_options->has_bias) { + bias.set_none(true); + } + + // Output quantization parameters + float output_scale_val = 0.05314; + ValueSpec output_scale(output_scale_val); + + const int32_t output_zero_point_val = + im2col_options == nullptr ? -1 : im2col_options->output_zero_point; + ValueSpec output_zero_point(output_zero_point_val); + + // Stride and padding parameters + ValueSpec stride({config.stride.h, config.stride.w}); + ValueSpec padding({config.padding.h, config.padding.w}); + + // Dilation and groups parameters + ValueSpec dilation({config.dilation.h, config.dilation.w}); + ValueSpec groups(config.groups); + + // Kernel size parameters + ValueSpec kernel_size({config.kernel.h, config.kernel.w}); + + // Output tensor (float/half) - [N, C_out, H_out, W_out] + ValueSpec output( + {config.batch, config.channels.out, H_out, W_out}, + input_dtype, + fp_storage_type, + fp_memory_layout, + DataGenType::ZEROS); + + // Add all specs to test case for q8ta_q8csw_q8to operation + test_case.add_input_spec(input_tensor); + test_case.add_input_spec(input_scale); + test_case.add_input_spec(input_zero_point); + test_case.add_input_spec(quantized_weight); + test_case.add_input_spec(weight_sums); + test_case.add_input_spec(weight_scales); + test_case.add_input_spec(output_scale); + test_case.add_input_spec(output_zero_point); + test_case.add_input_spec(bias); + test_case.add_input_spec(kernel_size); + test_case.add_input_spec(stride); + test_case.add_input_spec(padding); + test_case.add_input_spec(dilation); + test_case.add_input_spec(groups); + + // Activation (none = no activation) + ValueSpec activation = ValueSpec::make_string( + im2col_options == nullptr ? "none" : im2col_options->activation); + test_case.add_input_spec(activation); + + // Add memory layout parameter for the quantized tensors + ValueSpec input_layout_int(static_cast(input_int8_memory_layout)); + test_case.add_input_spec(input_layout_int); + + ValueSpec output_layout_int(static_cast(output_int8_memory_layout)); + test_case.add_input_spec(output_layout_int); + + // Add impl_selector string + ValueSpec impl_selector_spec = ValueSpec::make_string(impl_selector); + test_case.add_input_spec(impl_selector_spec); + + test_case.add_output_spec(output); + + test_case.set_abs_tolerance( + im2col_options == nullptr ? output_scale_val + 1e-4f + : output_scale_val * 0.25f); + + // Filter out quantize/dequantize shaders from timing measurements + test_case.set_shader_filter({ + "nchw_to", + "to_nchw", + "q8ta_quantize", + "q8ta_dequantize", + }); + + return test_case; +} + +} // namespace + +TestCase create_test_case_from_config( + const Conv2dConfig& config, + vkapi::ScalarType input_dtype, + utils::StorageType fp_storage_type, + utils::GPUMemoryLayout int8_memory_layout, + const std::string& impl_selector, + const Im2colUnsignedTestOptions* im2col_options, + const float input_scale_val, + const DataGenType input_data_gen) { + return create_test_case_from_config_with_layouts( + config, + input_dtype, + fp_storage_type, + int8_memory_layout, + int8_memory_layout, + impl_selector, + im2col_options, + input_scale_val, + input_data_gen); +} + +// SceneX route tests. The kPackedInt8_4C input + kPackedInt8_4W4C output +// layout combination is only exercised here (default generators never pair +// them); run via the scenex_regular and scenex_grouped sets. Zero +// tolerances are exact by construction: both pipelines accumulate in int32 +// with identical requantize, so any mismatch is a real regression, not noise. +TestCase create_scenex_test_case( + const Conv2dConfig& source_config, + const std::string& route) { + Conv2dConfig config = source_config; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = + make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); + + const std::string impl_selector = route == "auto" ? "" + : route == "direct" ? "general" + : "im2col"; + TestCase test_case = create_test_case_from_config_with_layouts( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + impl_selector, + /*im2col_options=*/nullptr, + /*input_scale_val=*/1.0f, + /*input_data_gen=*/DataGenType::RANDINT); + test_case.set_abs_tolerance(0.0f); + test_case.set_rel_tolerance(0.0f); + return test_case; +} + +// Custom FLOP calculator for quantized conv2d operation +int64_t quantized_conv2d_flop_calculator(const TestCase& test_case) { + int kernel_idx = 9; // kernel_size is at index 9 for q8ta_q8csw_q8to + + // Get input and weight dimensions + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); + + const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); + + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t C_out = output_sizes[1]; + int64_t K_h = kernel_sizes[0]; + int64_t K_w = kernel_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; + + // Calculate FLOPs for quantized conv2d operation + // Each output element requires: + // - C_in * K_h * K_w multiply-accumulate operations + // - Additional operations for quantization/dequantization + int64_t output_elements = N * C_out * H_out * W_out; + int64_t ops_per_output = C_in * K_h * K_w; + + int64_t flop = output_elements * (ops_per_output); + + return flop; +} + +namespace { + +// Reference implementation for activation, weight, and output quantized conv2d +void conv2d_q8ta_q8csw_q8to_reference_impl(TestCase& test_case) { + // Extract input specifications + int32_t idx = 0; + const ValueSpec& input_spec = test_case.inputs()[idx++]; + const ValueSpec& input_scale_spec = test_case.inputs()[idx++]; + const ValueSpec& input_zeros_spec = test_case.inputs()[idx++]; + const ValueSpec& weight_spec = test_case.inputs()[idx++]; + const ValueSpec& weight_sums_spec = test_case.inputs()[idx++]; + (void)weight_sums_spec; + const ValueSpec& weight_scales_spec = test_case.inputs()[idx++]; + const ValueSpec& output_scale_spec = test_case.inputs()[idx++]; + const ValueSpec& output_zeros_spec = test_case.inputs()[idx++]; + const ValueSpec& bias_spec = test_case.inputs()[idx++]; + const ValueSpec& kernel_size_spec = test_case.inputs()[idx++]; + const ValueSpec& stride_spec = test_case.inputs()[idx++]; + const ValueSpec& padding_spec = test_case.inputs()[idx++]; + const ValueSpec& dilation_spec = test_case.inputs()[idx++]; + const ValueSpec& groups_spec = test_case.inputs()[idx++]; + const ValueSpec& activation_spec = test_case.inputs()[idx++]; + const ValueSpec& layout_spec = test_case.inputs()[idx++]; + (void)layout_spec; // Not used in reference implementation + const ValueSpec& output_layout_spec = test_case.inputs()[idx++]; + (void)output_layout_spec; // Not used in reference implementation + const ValueSpec& impl_selector_spec = test_case.inputs()[idx++]; + (void)impl_selector_spec; // Not used in reference implementation + + // Extract output specification (mutable reference) + ValueSpec& output_spec = test_case.outputs()[0]; + + // Get tensor dimensions + auto input_sizes = input_spec.get_tensor_sizes(); // [N, C_in, H_in, W_in] + auto weight_sizes = + weight_spec.get_tensor_sizes(); // [C_out, C_in_per_group * K_h * K_w] + auto output_sizes = + output_spec.get_tensor_sizes(); // [N, C_out, H_out, W_out] + + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t H_in = input_sizes[2]; + int64_t W_in = input_sizes[3]; + int64_t C_out = output_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; + + // Get kernel dimensions from kernel_size ValueSpec + auto kernel_size_data = kernel_size_spec.get_int32_data(); + int64_t K_h = kernel_size_data[0]; + int64_t K_w = kernel_size_data[1]; + + // Get stride, padding, dilation, and groups + auto stride_data = stride_spec.get_int32_data(); + auto padding_data = padding_spec.get_int32_data(); + auto dilation_data = dilation_spec.get_int32_data(); + int64_t stride_h = stride_data[0]; + int64_t stride_w = stride_data[1]; + int64_t pad_h = padding_data[0]; + int64_t pad_w = padding_data[1]; + int64_t dilation_h = dilation_data[0]; + int64_t dilation_w = dilation_data[1]; + int64_t groups = groups_spec.get_int_value(); + + const int64_t reference_operations = + N * C_out * H_out * W_out * (C_in / groups) * K_h * K_w; + const bool has_large_dimension = N > kRefDimSizeLimit || + C_in > kRefDimSizeLimit || H_in > kRefDimSizeLimit || + W_in > kRefDimSizeLimit || C_out > kRefDimSizeLimit; + if (has_large_dimension && reference_operations > kRefOperationLimit) { + throw std::invalid_argument( + "One or more dimensions exceed the allowed limit for reference implementation."); + std::cout + << "Reference implementation: computation may take some time for large tensors..." + << std::endl; + } + + if (input_spec.dtype != vkapi::kFloat) { + throw std::invalid_argument("Unsupported dtype"); + } + + // Get raw data pointers + auto& input_data = input_spec.get_float_data(); + const float input_scale = input_scale_spec.get_float_value(); + const int32_t input_zero_point = input_zeros_spec.get_int_value(); + + auto& weight_data = weight_spec.get_int8_data(); + auto& weight_scales_data = weight_scales_spec.get_float_data(); + auto& bias_data = bias_spec.get_float_data(); + + const float output_scale = output_scale_spec.get_float_value(); + const int32_t output_zero_point = output_zeros_spec.get_int_value(); + + // Calculate channels per group for grouped convolution + int64_t C_in_per_group = C_in / groups; + int64_t C_out_per_group = C_out / groups; + + // Calculate number of output elements + int64_t num_output_elements = N * C_out * H_out * W_out; + + auto& ref_data = output_spec.get_ref_float_data(); + ref_data.resize(num_output_elements); + + const int64_t in_features = utils::align_up_4(C_in_per_group * K_h * K_w); + + // Perform activation, weight, and output quantized conv2d operation + for (int64_t n = 0; n < N; ++n) { + for (int64_t out_c = 0; out_c < C_out; ++out_c) { + for (int64_t out_h = 0; out_h < H_out; ++out_h) { + for (int64_t out_w = 0; out_w < W_out; ++out_w) { + int32_t int_sum = 0; + int32_t weight_sum = 0; // Track weight sum on the fly + + // Determine which group this output channel belongs to + int64_t group_idx = out_c / C_out_per_group; + int64_t in_c_start = group_idx * C_in_per_group; + int64_t in_c_end = (group_idx + 1) * C_in_per_group; + + // Convolution operation with integer accumulation + for (int64_t in_c = in_c_start; in_c < in_c_end; ++in_c) { + for (int64_t kh = 0; kh < K_h; ++kh) { + for (int64_t kw = 0; kw < K_w; ++kw) { + // Calculate input position with dilation + int64_t in_h = out_h * stride_h - pad_h + kh * dilation_h; + int64_t in_w = out_w * stride_w - pad_w + kw * dilation_w; + + // Check bounds (zero padding) + if (in_h >= 0 && in_h < H_in && in_w >= 0 && in_w < W_in) { + // Get input value and quantize to int8 + int64_t input_idx = n * (C_in * H_in * W_in) + + in_c * (H_in * W_in) + in_h * W_in + in_w; + + float quant_input_f = + std::round(input_data[input_idx] / input_scale) + + input_zero_point; + quant_input_f = + std::min(std::max(quant_input_f, -128.0f), 127.0f); + int8_t quantized_input = static_cast(quant_input_f); + + // Get quantized weight (already int8) + // Weight layout: [C_out, C_in_per_group * K_h * K_w] + int64_t weight_idx = out_c * in_features + + (kh * (K_w * C_in_per_group) + kw * C_in_per_group + + (in_c % C_in_per_group)); + int8_t quantized_weight = weight_data[weight_idx]; + + // Integer multiplication and accumulation + int_sum += static_cast(quantized_input) * + static_cast(quantized_weight); + + // Track weight sum for this output channel on the fly + weight_sum += static_cast(quantized_weight); + } else { + // For zero padding, we still need to account for the weight + // in weight_sum when input is effectively 0 (but quantized 0 + // is input_zero_point) + int64_t weight_idx = out_c * in_features + + (kh * (K_w * C_in_per_group) + kw * C_in_per_group + + (in_c % C_in_per_group)); + int8_t quantized_weight = weight_data[weight_idx]; + + // Add contribution from zero-padded input (quantized zero = + // input_zero_point) + int_sum += static_cast(input_zero_point) * + static_cast(quantized_weight); + + // Track weight sum for this output channel on the fly + weight_sum += static_cast(quantized_weight); + } + } + } + } + + // Convert accumulated integer result to float and apply scales + // Final result = (int_sum - zero_point_correction) * input_scale * + // weight_scale + bias zero_point_correction = input_zero_point * + // sum_of_weights_for_this_output_channel + int32_t zero_point_correction = input_zero_point * weight_sum; + int32_t accum_adjusted = int_sum - zero_point_correction; + float float_result = + accum_adjusted * input_scale * weight_scales_data[out_c]; + + if (!bias_spec.is_none()) { + float_result += bias_data[out_c]; + } + if (activation_spec.get_string_value() == "relu") { + float_result = std::max(float_result, 0.0f); + } + + // Quantize the output to int8 + float quant_output_f = + std::round(float_result / output_scale) + output_zero_point; + quant_output_f = std::min(std::max(quant_output_f, -128.0f), 127.0f); + int8_t quantized_output = static_cast(quant_output_f); + + // Dequantize back to float + float dequant_output = + (static_cast(quantized_output) - output_zero_point) * + output_scale; + + int64_t output_idx = n * (C_out * H_out * W_out) + + out_c * (H_out * W_out) + out_h * W_out + out_w; + ref_data[output_idx] = dequant_output; + } + } + } + } +} + +} // namespace + +void reference_impl(TestCase& test_case) { + conv2d_q8ta_q8csw_q8to_reference_impl(test_case); +} + +// The impl selector holds one of "", "general", or "im2col"; activation and +// other string inputs use disjoint values. Overwrite it by value so a +// reordered input list fails loudly instead of mutating the wrong spec. +// Note: for route == "direct" the measured run already forces "general", so +// this reference re-executes the identical implementation and only checks +// determinism; genuine cross-implementation correctness comes from the +// auto/im2col legs. +void scenex_direct_reference(TestCase& test_case) { + TestCase direct_case = test_case; + bool found_selector = false; + for (auto it = direct_case.inputs().rbegin(); + it != direct_case.inputs().rend(); + ++it) { + if (it->is_string() && + (it->get_string_value().empty() || + it->get_string_value() == "general" || + it->get_string_value() == "im2col")) { + it->string_data = "general"; + found_selector = true; + break; + } + } + if (!found_selector) { + throw std::runtime_error("scenex reference: impl selector input not found"); + } + execute_test_case( + direct_case, + /*warmup_runs=*/1, + /*benchmark_runs=*/1, + /*chained_dispatches=*/1, + /*write_outputs=*/true); + test_case.outputs().at(0).get_ref_float_data() = + direct_case.outputs().at(0).get_float_data(); +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.h new file mode 100644 index 00000000000..7d87b00f63f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d.h @@ -0,0 +1,81 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +constexpr int64_t kRefDimSizeLimit = 100; +constexpr int64_t kRefOperationLimit = 2 * 1024 * 1024; + +struct Im2colUnsignedTestOptions { + int32_t input_zero_point = 2; + int32_t output_zero_point = -1; + bool use_extreme_values = false; + bool use_accumulator_limit_values = false; + bool has_bias = true; + const char* activation = "none"; + float weight_scale = 1.0f / 256.0f; +}; + +struct PointwiseTestOptions { + int32_t input_zero_point = 2; + bool has_bias = true; +}; + +TestCase create_test_case_from_config( + const Conv2dConfig& config, + vkapi::ScalarType input_dtype, + utils::StorageType fp_storage_type, + utils::GPUMemoryLayout int8_memory_layout, + const std::string& impl_selector = "", + const Im2colUnsignedTestOptions* im2col_options = nullptr, + const float input_scale_val = 0.008123f, + const DataGenType input_data_gen = DataGenType::RANDOM); + +TestCase create_scenex_test_case( + const Conv2dConfig& source_config, + const std::string& route); + +TestCase create_dw_test_case_from_config( + const Conv2dConfig& config, + vkapi::ScalarType input_dtype, + utils::StorageType fp_storage_type, + utils::GPUMemoryLayout int8_memory_layout, + const std::string& impl_selector = ""); + +TestCase create_pw_test_case_from_config( + const Conv2dConfig& config, + vkapi::ScalarType input_dtype, + utils::StorageType fp_storage_type, + utils::GPUMemoryLayout int8_memory_layout, + const std::string& impl_selector = "", + const PointwiseTestOptions& options = {}); + +void reference_impl(TestCase& test_case); + +void scenex_direct_reference(TestCase& test_case); + +void dw_reference_impl(TestCase& test_case); + +void pw_reference_impl(TestCase& test_case); + +int64_t quantized_conv2d_flop_calculator(const TestCase& test_case); + +int64_t quantized_conv2d_dw_flop_calculator(const TestCase& test_case); + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_dw.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d_dw.cpp similarity index 62% rename from backends/vulkan/test/custom_ops/test_q8ta_conv2d_dw.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d_dw.cpp index c573b8b5d39..8682971620d 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_dw.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d_dw.cpp @@ -4,34 +4,27 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include - -#include -#include - -#include - -#include "conv2d_utils.h" -#include "utils.h" - -// #define DEBUG_MODE - -using namespace executorch::vulkan::prototyping; +#include -using namespace vkcompute; +#include +#include +#include +#include +#include -static constexpr int64_t kRefDimSizeLimit = 100; -static constexpr int64_t kRefOperationLimit = 2 * 1024 * 1024; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { // Utility function to create a test case from a Conv2dConfig for depthwise // convolution -TestCase create_test_case_from_config( +TestCase create_dw_test_case_from_config( const Conv2dConfig& config, vkapi::ScalarType input_dtype, utils::StorageType fp_storage_type, utils::GPUMemoryLayout int8_memory_layout, - const std::string& impl_selector = "") { + const std::string& impl_selector) { TestCase test_case; // Calculate output dimensions @@ -224,269 +217,38 @@ TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for quantized depthwise conv2d operation (for -// debugging) -std::vector generate_quantized_conv2d_dw_easy_cases() { - std::vector test_cases; - - // Single simple configuration for debugging - depthwise convolution - Conv2dConfig config = { - OutInChannels(8, 8), // channels (out, in) - equal for depthwise - InputSize2D(8, 8), // input_size (h, w) - KernelSize(3, 3), // kernel - Stride(1, 1), // stride - Padding(1, 1), // padding - Dilation(1, 1), // dilation - 8, // groups = channels.out for depthwise - }; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - - std::vector fp_storage_types = {utils::kTexture3D}; - - // Memory layouts for int8 tensors - test both optimized (4W4C) and general - // paths - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - // Generate test cases for each combination - for (const utils::StorageType fp_storage_type : fp_storage_types) { - for (const utils::GPUMemoryLayout int8_memory_layout : - int8_memory_layouts) { - config.test_case_name = - make_test_case_name(config, false, fp_storage_type, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); - - // For 4W4C layout, also test the legacy implementation - if (int8_memory_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"legacy_4w4c")); - } - } - } - - return test_cases; -} +// Custom FLOP calculator for quantized depthwise conv2d operation +int64_t quantized_conv2d_dw_flop_calculator(const TestCase& test_case) { + int kernel_idx = 9; // kernel_size is at index 9 for q8ta_q8csw_q8to -std::vector generate_quantized_conv2d_dw_narrow_workgroup_cases() { - std::vector test_cases; - std::vector configs = { - {OutInChannels(128, 128), - InputSize2D(7, 7), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 128}, - {OutInChannels(64, 64), - InputSize2D(9, 9), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 64}, - {OutInChannels(64, 64), - InputSize2D(13, 13), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 64}, - }; - - for (auto& config : configs) { - const bool is_performance = config.channels.out > kRefDimSizeLimit; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = make_test_case_name( - config, is_performance, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4C)); - } - return test_cases; -} + // Get input and weight dimensions + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); -// Generate test cases for quantized depthwise conv2d operation -std::vector generate_quantized_conv2d_dw_test_cases() { - std::vector test_cases; - if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { - return test_cases; - } + const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); - std::vector configs = { - // Depthwise convolutions: groups = channels.out, channels.in = - // channels.out - {OutInChannels(32, 32), - InputSize2D(64, 64), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 32}, - {OutInChannels(64, 64), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 64}, - {OutInChannels(64, 64), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 64}, - {OutInChannels(80, 80), - InputSize2D(16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 80}, - {OutInChannels(16, 16), - InputSize2D(57, 33), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 16}, - // Different kernel sizes for depthwise - {OutInChannels(32, 32), - InputSize2D(64, 64), - KernelSize(5, 5), - Stride(1, 1), - Padding(2, 2), - Dilation(1, 1), - 32}, - {OutInChannels(96, 96), - InputSize2D(64, 64), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 96}, - // Performance cases - {OutInChannels(128, 128), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 128}, - {OutInChannels(64, 64), - InputSize2D(256, 256), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 64}, - {OutInChannels(288, 288), - InputSize2D(16, 16), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 288}, - {OutInChannels(32, 32), - InputSize2D(128, 128), - KernelSize(3, 3), - Stride(1, 1), - Padding(2, 2), - Dilation(1, 1), - 32}}; - - // Test with different storage types and data types - std::vector fp_storage_types = {utils::kTexture3D}; - - // Memory layouts for int8 tensors - test both optimized (4W4C) and general - // paths - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - // Generate test cases for each combination - for (auto& config : configs) { - bool is_performance = config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit || - config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - - config.op_name = "conv2d_q8ta_q8csw_q8to"; - - for (const utils::StorageType fp_storage_type : fp_storage_types) { - for (const utils::GPUMemoryLayout int8_memory_layout : - int8_memory_layouts) { - config.test_case_name = make_test_case_name( - config, is_performance, fp_storage_type, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); - - // For 4W4C layout, also test the legacy implementation - if (int8_memory_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"legacy_4w4c")); - } - } - } - } + int64_t N = input_sizes[0]; + int64_t C_out = output_sizes[1]; + int64_t K_h = kernel_sizes[0]; + int64_t K_w = kernel_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; - std::vector batch_configs = { - {OutInChannels(8, 8), - InputSize2D(8, 8), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 8, - 2}, - {OutInChannels(128, 128), - InputSize2D(64, 64), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 128, - 1}, - {OutInChannels(128, 128), - InputSize2D(64, 64), - KernelSize(5, 5), - Stride(2, 2), - Padding(2, 2), - Dilation(1, 1), - 128, - 60}}; - - for (auto& config : batch_configs) { - const bool is_performance = config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = make_test_case_name( - config, is_performance, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4C1W)); - if (config.batch == 2) { - test_cases.push_back(create_test_case_from_config( - config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4W4C)); - } - } + // Calculate FLOPs for quantized depthwise conv2d operation + // Each output element requires: + // - K_h * K_w multiply-accumulate operations (only one input channel per + // output channel) + // - Additional operations for quantization/dequantization + int64_t output_elements = N * C_out * H_out * W_out; + int64_t ops_per_output = K_h * K_w; - auto narrow_workgroup_cases = - generate_quantized_conv2d_dw_narrow_workgroup_cases(); - test_cases.insert( - test_cases.end(), - narrow_workgroup_cases.begin(), - narrow_workgroup_cases.end()); + int64_t flop = output_elements * ops_per_output; - return test_cases; + return flop; } +namespace { + // Reference implementation for activation, weight, and output quantized // depthwise conv2d void conv2d_q8ta_q8csw_q8to_dw_reference_impl(TestCase& test_case) { @@ -681,80 +443,13 @@ void conv2d_q8ta_q8csw_q8to_dw_reference_impl(TestCase& test_case) { } } -void reference_impl(TestCase& test_case) { - conv2d_q8ta_q8csw_q8to_dw_reference_impl(test_case); -} - -// Custom FLOP calculator for quantized depthwise conv2d operation -int64_t quantized_conv2d_dw_flop_calculator(const TestCase& test_case) { - int kernel_idx = 9; // kernel_size is at index 9 for q8ta_q8csw_q8to - - // Get input and weight dimensions - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - - const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); - - int64_t N = input_sizes[0]; - int64_t C_out = output_sizes[1]; - int64_t K_h = kernel_sizes[0]; - int64_t K_w = kernel_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - // Calculate FLOPs for quantized depthwise conv2d operation - // Each output element requires: - // - K_h * K_w multiply-accumulate operations (only one input channel per - // output channel) - // - Additional operations for quantization/dequantization - int64_t output_elements = N * C_out * H_out * W_out; - int64_t ops_per_output = K_h * K_w; - - int64_t flop = output_elements * ops_per_output; +} // namespace - return flop; +void dw_reference_impl(TestCase& test_case) { + conv2d_q8ta_q8csw_q8to_dw_reference_impl(test_case); } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "Quantized Depthwise Conv2d Operation with Output Quantization Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - bool narrow_workgroups_only = false; - for (int i = 1; i < argc; ++i) { - const std::string arg(argv[i]); - if (arg == "--narrow-workgroups-only") { - narrow_workgroups_only = true; - } else { - std::cerr << "Unknown argument: " << arg << std::endl; - return 2; - } - } -#ifdef DEBUG_MODE - auto test_case_generator = generate_quantized_conv2d_dw_easy_cases; -#else - auto test_case_generator = generate_quantized_conv2d_dw_test_cases; -#endif - if (narrow_workgroups_only) { - test_case_generator = generate_quantized_conv2d_dw_narrow_workgroup_cases; - } - - auto results = execute_test_cases( - test_case_generator, - quantized_conv2d_dw_flop_calculator, - "QuantizedDepthwiseInt8Conv2d", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_pw.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d_pw.cpp similarity index 61% rename from backends/vulkan/test/custom_ops/test_q8ta_conv2d_pw.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d_pw.cpp index 1003c0540ce..244825c7525 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_pw.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/conv2d_pw.cpp @@ -4,40 +4,28 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. +#include + #include +#include #include +#include +#include #include -#include -#include -#include - -#include - -#include "conv2d_utils.h" -#include "utils.h" - -// #define DEBUG_MODE - -using namespace executorch::vulkan::prototyping; - -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 100; - -struct PointwiseTestOptions { - int32_t input_zero_point = 2; - bool has_bias = true; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { // Utility function to create a test case from a Conv2dConfig -static TestCase create_test_case_from_config( +TestCase create_pw_test_case_from_config( const Conv2dConfig& config, vkapi::ScalarType input_dtype, utils::StorageType fp_storage_type, utils::GPUMemoryLayout int8_memory_layout, - const std::string& impl_selector = "", - const PointwiseTestOptions& options = {}) { + const std::string& impl_selector, + const PointwiseTestOptions& options) { TestCase test_case; // Calculate output dimensions @@ -86,12 +74,7 @@ static TestCase create_test_case_from_config( input_dtype, fp_storage_type, fp_memory_layout, -#ifdef DEBUG_MODE - DataGenType::RANDOM -#else - DataGenType::RANDOM -#endif - ); + DataGenType::RANDOM); if (debugging()) { print_valuespec_data(input_tensor, "input_tensor"); @@ -233,228 +216,10 @@ static TestCase create_test_case_from_config( return test_case; } -// Generate test cases for quantized pointwise conv2d operation. When -// pw_selector is non-empty ("pw_signed", "pw_unsigned", "pw_auto"), every -// non-legacy case carries that selector so the test graph builder can force or -// verify the unsigned-dot routing instead of using the default automatic path. -static std::vector generate_quantized_conv2d_pw_test_cases( - const std::string& pw_selector = "") { - std::vector test_cases; - if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { - return test_cases; - } - - std::vector configs = { - // OC < 4 cases to test edge cases with partial output channel blocks - {OutInChannels(1, 16), - InputSize2D(8, 8), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(2, 16), - InputSize2D(8, 8), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(3, 16), - InputSize2D(8, 8), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - // Pointwise convolutions: kernel size 1x1 - {OutInChannels(32, 3), - InputSize2D(64, 64), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(64, 32), - InputSize2D(32, 32), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(96, 64), - InputSize2D(16, 16), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(13, 7), - InputSize2D(57, 33), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(80, 40), - InputSize2D(64, 64), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - // Performance cases (pointwise - will use im2col) - {OutInChannels(160, 480), - InputSize2D(8, 8), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(22, 48), - InputSize2D(256, 256), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(48, 48), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(128, 128), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - {OutInChannels(64, 576), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}, - }; - - // Test with different storage types and memory layouts - std::vector fp_storage_types = {utils::kTexture3D}; - - // Memory layouts for int8 tensors - test both optimized (4W4C) and general - // paths - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - // Generate test cases for each combination - for (auto& config : configs) { - bool is_performance = config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit || - config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - - config.op_name = "conv2d_q8ta_q8csw_q8to"; - - for (const utils::StorageType fp_storage_type : fp_storage_types) { - for (const utils::GPUMemoryLayout int8_memory_layout : - int8_memory_layouts) { - config.test_case_name = make_test_case_name( - config, is_performance, fp_storage_type, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - pw_selector)); - - // For 4W4C layout, also test the legacy implementation - if (int8_memory_layout == utils::kPackedInt8_4W4C) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - fp_storage_type, - int8_memory_layout, - /*impl_selector=*/"legacy_4w4c")); - } - } - } - } - - std::vector batch_configs = { - {OutInChannels(8, 8), - InputSize2D(8, 8), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1, - 2}, - {OutInChannels(64, 32), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1, - 1}, - {OutInChannels(64, 32), - InputSize2D(128, 128), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1, - 60}}; - - for (auto& config : batch_configs) { - const bool is_performance = config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - config.op_name = "conv2d_q8ta_q8csw_q8to"; - config.test_case_name = make_test_case_name( - config, is_performance, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4C1W, - pw_selector)); - if (config.batch == 2) { - test_cases.push_back(create_test_case_from_config( - config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - pw_selector)); - } - } - - Conv2dConfig edge_config{ - OutInChannels(13, 7), - InputSize2D(7, 5), - KernelSize(1, 1), - Stride(1, 1), - Padding(0, 0), - Dilation(1, 1), - 1}; - edge_config.op_name = "conv2d_q8ta_q8csw_q8to"; - edge_config.test_case_name = make_test_case_name( - edge_config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - edge_config, - vkapi::kFloat, - utils::kTexture3D, - utils::kPackedInt8_4W4C, - pw_selector, - {.input_zero_point = -128, .has_bias = false})); - - return test_cases; -} +namespace { // Reference implementation for activation, weight, and output quantized conv2d -static void conv2d_q8ta_q8csw_q8to_reference_impl(TestCase& test_case) { +void conv2d_q8ta_q8csw_q8to_pw_reference_impl(TestCase& test_case) { // Extract input specifications int32_t idx = 0; const ValueSpec& input_spec = test_case.inputs()[idx++]; @@ -656,93 +421,13 @@ static void conv2d_q8ta_q8csw_q8to_reference_impl(TestCase& test_case) { } } -static void reference_impl(TestCase& test_case) { - conv2d_q8ta_q8csw_q8to_reference_impl(test_case); -} - -// Custom FLOP calculator for quantized pointwise conv2d operation -static int64_t quantized_conv2d_flop_calculator(const TestCase& test_case) { - int kernel_idx = 9; // kernel_size is at index 9 for q8ta_q8csw_q8to - - // Get input and weight dimensions - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); +} // namespace - const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t C_out = output_sizes[1]; - int64_t K_h = kernel_sizes[0]; - int64_t K_w = kernel_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - // Calculate FLOPs for quantized conv2d operation - // Each output element requires: - // - C_in * K_h * K_w multiply-accumulate operations - // - Additional operations for quantization/dequantization - int64_t output_elements = N * C_out * H_out * W_out; - int64_t ops_per_output = C_in * K_h * K_w; - - int64_t flop = output_elements * (ops_per_output); - - return flop; +void pw_reference_impl(TestCase& test_case) { + conv2d_q8ta_q8csw_q8to_pw_reference_impl(test_case); } -int main(int argc, char* argv[]) { - const vkapi::Adapter& adapter = *vkcompute::api::context()->adapter_ptr(); - const bool prefers_unsigned_dot = - adapter.accelerates_unsigned_packed4x8_dot() && - !adapter.accelerates_signed_packed4x8_dot(); - VK_CHECK_COND( - can_use_unsigned_pw_dot(adapter, kMaxUnsignedDotAccumulatorBytes) == - prefers_unsigned_dot); - VK_CHECK_COND( - !can_use_unsigned_pw_dot(adapter, kMaxUnsignedDotAccumulatorBytes + 1)); - - std::string pw_impl_selector; - for (int i = 1; i < argc; ++i) { - const std::string arg(argv[i]); - if (arg == "--pw-path=signed") { - pw_impl_selector = "pw_signed"; - } else if (arg == "--pw-path=unsigned") { - pw_impl_selector = "pw_unsigned"; - } else if (arg == "--pw-path=auto") { - pw_impl_selector = "pw_auto"; - } else { - std::cerr << "Unknown argument: " << arg << std::endl; - return 2; - } - } - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(true); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "Quantized Pointwise Conv2d (1x1) Operation Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - // Execute test cases using the new framework with custom FLOP calculator - const auto test_case_generator = [pw_impl_selector]() { - return generate_quantized_conv2d_pw_test_cases(pw_impl_selector); - }; - auto results = execute_test_cases( - test_case_generator, - quantized_conv2d_flop_calculator, - "QuantizedConv2dPW", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/conv2d_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/conv2d_sets.cpp new file mode 100644 index 00000000000..cdabdc58969 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/conv2d_sets.cpp @@ -0,0 +1,319 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +std::vector generate_config_test_cases( + const std::vector& configs, + const std::vector& batch_configs, + const std::vector& fp_storage_types, + const std::vector& int8_memory_layouts) { + std::vector test_cases; + if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { + return test_cases; + } + + // Generate test cases for each combination + for (auto config : configs) { + bool is_performance = config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit || + config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + + config.op_name = "conv2d_q8ta_q8csw_q8to"; + + for (const utils::StorageType fp_storage_type : fp_storage_types) { + for (const utils::GPUMemoryLayout int8_memory_layout : + int8_memory_layouts) { + config.test_case_name = make_test_case_name( + config, is_performance, fp_storage_type, utils::kBuffer); + + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"general")); + + // Test im2col implementation when input channels per group is a + // multiple of 4 + const int64_t in_channels_per_group = + config.channels.in / config.groups; + if (in_channels_per_group % 4 == 0) { + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"im2col")); + } + + // For 4W4C layout, also test the legacy implementation + if (int8_memory_layout == utils::kPackedInt8_4W4C) { + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"legacy_4w4c")); + } + + test_cases.push_back(create_test_case_from_config( + config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); + } + } + } + + for (auto config : batch_configs) { + const bool is_performance = config.batch > kRefDimSizeLimit || + config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit || + config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = make_test_case_name( + config, is_performance, utils::kTexture3D, utils::kBuffer); + test_cases.push_back(create_test_case_from_config( + config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4C1W)); + if (config.batch == 2) { + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4C1W, + /*impl_selector=*/"im2col")); + test_cases.push_back(create_test_case_from_config( + config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4W4C)); + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4W4C, + /*impl_selector=*/"im2col")); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "correctness") { + const std::vector configs = { + // General 2D convolutions + {OutInChannels(32, 3), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(32, 3), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(8, 8), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(16, 32), + InputSize2D(77, 77), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + // Grouped convolutions + {OutInChannels(64, 32), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2}, + {OutInChannels(96, 96), + InputSize2D(81, 81), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 3}, + {OutInChannels(96, 96), + InputSize2D(64, 64), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 4}, + }; + const std::vector batch_configs = { + {OutInChannels(16, 32), + InputSize2D(7, 7), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 2}, + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + return { + generate_config_test_cases( + configs, batch_configs, fp_storage_types, int8_memory_layouts), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "performance") { + const std::vector configs = { + // Performance cases (3x3 convs - will use im2col) + {OutInChannels(32, 3), + InputSize2D(256, 256), + KernelSize(3, 3), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(64, 64), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + // Performance cases (grouped convs) + {OutInChannels(64, 64), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2}, + {OutInChannels(96, 96), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 3}, + {OutInChannels(128, 128), + InputSize2D(128, 128), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 4}, + // SceneX v9 grouped convolutions (large spatial) + {OutInChannels(128, 128), + InputSize2D(256, 256), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 4}, + {OutInChannels(64, 64), + InputSize2D(256, 256), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2}, + // Deep channels + small spatial (ResNet50 stage 5 bottleneck) + {OutInChannels(512, 512), + InputSize2D(7, 7), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + // Strided 1x1 shortcut (worst-case strided downsample) + {OutInChannels(2048, 1024), + InputSize2D(14, 14), + KernelSize(1, 1), + Stride(2, 2), + Padding(0, 0), + Dilation(1, 1), + 1}, + }; + const std::vector batch_configs = { + {OutInChannels(32, 3), + InputSize2D(256, 256), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1, + 1}, + {OutInChannels(32, 3), + InputSize2D(256, 256), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1, + 60}, + {OutInChannels(512, 256), + InputSize2D(10, 13), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1, + 60}, + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + return { + generate_config_test_cases( + configs, batch_configs, fp_storage_types, int8_memory_layouts), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/debug_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/debug_set.cpp new file mode 100644 index 00000000000..255f92add31 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/debug_set.cpp @@ -0,0 +1,88 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +// Generate easy test cases for quantized conv2d operation (for debugging) +std::vector generate_quantized_conv2d_easy_cases( + const Conv2dConfig& source_config, + const std::vector& fp_storage_types, + const std::vector& int8_memory_layouts) { + std::vector test_cases; + Conv2dConfig config = source_config; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + + // Generate test cases for each combination + for (const utils::StorageType fp_storage_type : fp_storage_types) { + for (const utils::GPUMemoryLayout int8_memory_layout : + int8_memory_layouts) { + config.test_case_name = + make_test_case_name(config, false, fp_storage_type, utils::kBuffer); + test_cases.push_back(create_test_case_from_config( + config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); + + // Test im2col implementation when input channels per group is a + // multiple of 4 + if ((config.channels.in / config.groups) % 4 == 0) { + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"im2col")); + } + // For 4W4C layout, also test the legacy implementation + if (int8_memory_layout == utils::kPackedInt8_4W4C) { + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"legacy_4w4c")); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "debug") { + // Single simple configuration for debugging + const Conv2dConfig config = { + OutInChannels(16, 8), // channels (out, in) + InputSize2D(5, 5), // input_size (h, w) + KernelSize(3, 3), // kernel + Stride(1, 1), // stride + Padding(1, 1), // padding + Dilation(1, 1), // dilation + 1, // groups + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + return { + generate_quantized_conv2d_easy_cases( + config, fp_storage_types, int8_memory_layouts), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_debug_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_debug_set.cpp new file mode 100644 index 00000000000..b5395dda31c --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_debug_set.cpp @@ -0,0 +1,79 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +// Generate easy test cases for quantized depthwise conv2d operation (for +// debugging) +std::vector generate_quantized_conv2d_dw_easy_cases( + const Conv2dConfig& source_config, + const std::vector& fp_storage_types, + const std::vector& int8_memory_layouts) { + std::vector test_cases; + Conv2dConfig config = source_config; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + + // Generate test cases for each combination + for (const utils::StorageType fp_storage_type : fp_storage_types) { + for (const utils::GPUMemoryLayout int8_memory_layout : + int8_memory_layouts) { + config.test_case_name = + make_test_case_name(config, false, fp_storage_type, utils::kBuffer); + test_cases.push_back(create_dw_test_case_from_config( + config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); + + // For 4W4C layout, also test the legacy implementation + if (int8_memory_layout == utils::kPackedInt8_4W4C) { + test_cases.push_back(create_dw_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"legacy_4w4c")); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "dw_debug") { + // Single simple configuration for debugging - depthwise convolution + const Conv2dConfig config = { + OutInChannels(8, 8), // channels (out, in) - equal for depthwise + InputSize2D(8, 8), // input_size (h, w) + KernelSize(3, 3), // kernel + Stride(1, 1), // stride + Padding(1, 1), // padding + Dilation(1, 1), // dilation + 8, // groups = channels.out for depthwise + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + return { + generate_quantized_conv2d_dw_easy_cases( + config, fp_storage_types, int8_memory_layouts), + dw_reference_impl, + quantized_conv2d_dw_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_narrow_workgroup_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_narrow_workgroup_set.cpp new file mode 100644 index 00000000000..49ba8573915 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_narrow_workgroup_set.cpp @@ -0,0 +1,68 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +std::vector generate_quantized_conv2d_dw_narrow_workgroup_cases( + const std::vector& configs) { + std::vector test_cases; + for (Conv2dConfig config : configs) { + const bool is_performance = config.channels.out > kRefDimSizeLimit; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = make_test_case_name( + config, is_performance, utils::kTexture3D, utils::kBuffer); + test_cases.push_back(create_dw_test_case_from_config( + config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4C)); + } + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "dw_narrow_workgroup") { + const std::vector configs = { + {OutInChannels(128, 128), + InputSize2D(7, 7), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 128}, + {OutInChannels(64, 64), + InputSize2D(9, 9), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 64}, + {OutInChannels(64, 64), + InputSize2D(13, 13), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 64}, + }; + return { + generate_quantized_conv2d_dw_narrow_workgroup_cases(configs), + dw_reference_impl, + quantized_conv2d_dw_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_sets.cpp new file mode 100644 index 00000000000..37a9d4e0cf6 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/dw_sets.cpp @@ -0,0 +1,216 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +std::vector generate_dw_test_cases( + const std::vector& configs, + const std::vector& batch_configs, + const std::vector& fp_storage_types, + const std::vector& int8_memory_layouts) { + std::vector test_cases; + if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { + return test_cases; + } + + // Generate test cases for each combination + for (auto config : configs) { + bool is_performance = config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit || + config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + + config.op_name = "conv2d_q8ta_q8csw_q8to"; + + for (const utils::StorageType fp_storage_type : fp_storage_types) { + for (const utils::GPUMemoryLayout int8_memory_layout : + int8_memory_layouts) { + config.test_case_name = make_test_case_name( + config, is_performance, fp_storage_type, utils::kBuffer); + test_cases.push_back(create_dw_test_case_from_config( + config, vkapi::kFloat, fp_storage_type, int8_memory_layout)); + + // For 4W4C layout, also test the legacy implementation + if (int8_memory_layout == utils::kPackedInt8_4W4C) { + test_cases.push_back(create_dw_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"legacy_4w4c")); + } + } + } + } + + for (auto config : batch_configs) { + const bool is_performance = config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = make_test_case_name( + config, is_performance, utils::kTexture3D, utils::kBuffer); + test_cases.push_back(create_dw_test_case_from_config( + config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4C1W)); + if (config.batch == 2) { + test_cases.push_back(create_dw_test_case_from_config( + config, vkapi::kFloat, utils::kTexture3D, utils::kPackedInt8_4W4C)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "dw_correctness") { + const std::vector configs = { + // Depthwise convolutions: groups = channels.out, channels.in = + // channels.out + {OutInChannels(32, 32), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 32}, + {OutInChannels(64, 64), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 64}, + {OutInChannels(64, 64), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 64}, + {OutInChannels(80, 80), + InputSize2D(16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 80}, + {OutInChannels(16, 16), + InputSize2D(57, 33), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 16}, + // Different kernel sizes for depthwise + {OutInChannels(32, 32), + InputSize2D(64, 64), + KernelSize(5, 5), + Stride(1, 1), + Padding(2, 2), + Dilation(1, 1), + 32}, + {OutInChannels(96, 96), + InputSize2D(64, 64), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 96}, + }; + const std::vector batch_configs = { + {OutInChannels(8, 8), + InputSize2D(8, 8), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 8, + 2}, + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + return { + generate_dw_test_cases( + configs, batch_configs, fp_storage_types, int8_memory_layouts), + dw_reference_impl, + quantized_conv2d_dw_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "dw_performance") { + const std::vector configs = { + // Performance cases + {OutInChannels(128, 128), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 128}, + {OutInChannels(64, 64), + InputSize2D(256, 256), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 64}, + {OutInChannels(288, 288), + InputSize2D(16, 16), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 288}, + {OutInChannels(32, 32), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(2, 2), + Dilation(1, 1), + 32}, + }; + const std::vector batch_configs = { + {OutInChannels(128, 128), + InputSize2D(64, 64), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 128, + 1}, + {OutInChannels(128, 128), + InputSize2D(64, 64), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 128, + 60}, + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + return { + generate_dw_test_cases( + configs, batch_configs, fp_storage_types, int8_memory_layouts), + dw_reference_impl, + quantized_conv2d_dw_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/im2col_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/im2col_set.cpp new file mode 100644 index 00000000000..df7ccce3f52 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/im2col_set.cpp @@ -0,0 +1,206 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +TestCase create_im2col_test_case( + const Conv2dConfig& source_config, + utils::StorageType fp_storage_type, + const std::string& impl_selector, + const Im2colUnsignedTestOptions& options) { + Conv2dConfig config = source_config; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = + make_test_case_name(config, false, fp_storage_type, utils::kBuffer); + return create_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + utils::kPackedInt8_4W4C, + impl_selector, + &options); +} + +std::vector generate_im2col_test_cases( + const std::vector>& + configs, + const std::vector& impl_selectors, + const std::vector& auto_configs, + const std::vector& buffer_output_configs) { + const vkapi::Adapter& adapter = *vkcompute::api::context()->adapter_ptr(); + std::vector test_cases; + for (const std::string& impl_selector : impl_selectors) { + if (impl_selector == "im2col_unsigned" && + !adapter.supports_int8_dot_product()) { + continue; + } + + for (const auto& [config, options] : configs) { + test_cases.push_back(create_im2col_test_case( + config, utils::kTexture3D, impl_selector, options)); + } + + if (impl_selector == "im2col_auto") { + for (const Conv2dConfig& config : auto_configs) { + test_cases.push_back(create_im2col_test_case( + config, + utils::kTexture3D, + impl_selector, + Im2colUnsignedTestOptions{})); + } + } + + if (impl_selector == "im2col_unsigned" || + (impl_selector == "im2col_auto" && + can_use_unsigned_pw_dot(adapter, 4))) { + for (Conv2dConfig config : buffer_output_configs) { + config.channels.out = utils::safe_downcast( + static_cast(adapter.max_texture2d_dim()) * 4 + + config.channels.out); + test_cases.push_back(create_im2col_test_case( + config, + utils::kBuffer, + impl_selector, + Im2colUnsignedTestOptions{})); + } + } + } + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "im2col") { + const std::vector> + configs = { + {{OutInChannels(5, 4), + InputSize2D(5, 5), + KernelSize(3, 3), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {.input_zero_point = 2, + .output_zero_point = -1, + .use_extreme_values = true, + .use_accumulator_limit_values = false, + .has_bias = true, + .activation = "none"}}, + {{OutInChannels(8, 8), + InputSize2D(5, 5), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {.input_zero_point = -7, + .output_zero_point = 3, + .use_extreme_values = true, + .use_accumulator_limit_values = false, + .has_bias = false, + .activation = "none"}}, + {{OutInChannels(12, 8), + InputSize2D(7, 7), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + {.input_zero_point = 127, + .output_zero_point = -5, + .use_extreme_values = true, + .use_accumulator_limit_values = false, + .has_bias = true, + .activation = "relu"}}, + {{OutInChannels(12, 8), + InputSize2D(9, 9), + KernelSize(3, 3), + Stride(1, 1), + Padding(2, 2), + Dilation(2, 2), + 1}, + {.input_zero_point = -128, + .output_zero_point = 5, + .use_extreme_values = true, + .use_accumulator_limit_values = false, + .has_bias = false, + .activation = "none"}}, + {{OutInChannels(8, 8), + InputSize2D(6, 7), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2}, + {.input_zero_point = 11, + .output_zero_point = -3, + .use_extreme_values = true, + .use_accumulator_limit_values = false, + .has_bias = true, + .activation = "relu"}}, + {{OutInChannels(1, 4), + InputSize2D(90, 91), + KernelSize(90, 91), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {.input_zero_point = 0, + .output_zero_point = -1, + .use_extreme_values = false, + .use_accumulator_limit_values = true, + .has_bias = true, + .activation = "none", + .weight_scale = 1.0f / 1000000.0f}}, + }; + const std::vector impl_selectors = { + "im2col", "im2col_unsigned", "im2col_auto"}; + const std::vector auto_configs = { + {OutInChannels(1, 4), + InputSize2D(91, 91), + KernelSize(91, 91), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + }; + // Output channels are offset by max_texture2d_dim * 4 so that the output + // exceeds the texture limit and uses buffer storage. + const std::vector buffer_output_configs = { + {OutInChannels(1, 4), + InputSize2D(1, 1), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + }; + return { + generate_im2col_test_cases( + configs, impl_selectors, auto_configs, buffer_output_configs), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/narrow_workgroup_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/narrow_workgroup_set.cpp new file mode 100644 index 00000000000..896b9ff66e5 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/narrow_workgroup_set.cpp @@ -0,0 +1,65 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +std::vector generate_narrow_workgroup_test_cases( + const std::vector& configs) { + std::vector test_cases; + for (Conv2dConfig config : configs) { + const bool is_performance = config.channels.out > kRefDimSizeLimit; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = make_test_case_name( + config, is_performance, utils::kTexture3D, utils::kBuffer); + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4C, + /*impl_selector=*/"general")); + } + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "narrow_workgroup") { + const std::vector configs = { + {OutInChannels(64, 32), + InputSize2D(9, 9), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + {OutInChannels(128, 32), + InputSize2D(7, 7), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + }; + return { + generate_narrow_workgroup_test_cases(configs), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/pw_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/pw_sets.cpp new file mode 100644 index 00000000000..046e8f6d271 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/pw_sets.cpp @@ -0,0 +1,294 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +// Generate test cases for quantized pointwise conv2d operation. A non-empty pw +// selector lets the test graph builder force or verify the unsigned-dot +// routing instead of the default automatic path. +std::vector generate_quantized_conv2d_pw_test_cases( + const std::vector& configs, + const std::vector& batch_configs, + const std::vector>& + edge_configs, + const std::vector& fp_storage_types, + const std::vector& int8_memory_layouts, + const std::vector& pw_selectors) { + std::vector test_cases; + if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { + return test_cases; + } + + // Generate test cases for each combination + for (Conv2dConfig config : configs) { + bool is_performance = config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit || + config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + + config.op_name = "conv2d_q8ta_q8csw_q8to"; + + for (const utils::StorageType fp_storage_type : fp_storage_types) { + for (const utils::GPUMemoryLayout int8_memory_layout : + int8_memory_layouts) { + config.test_case_name = make_test_case_name( + config, is_performance, fp_storage_type, utils::kBuffer); + for (const std::string& pw_selector : pw_selectors) { + test_cases.push_back(create_pw_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + pw_selector)); + } + + // For 4W4C layout, also test the legacy implementation + if (int8_memory_layout == utils::kPackedInt8_4W4C) { + test_cases.push_back(create_pw_test_case_from_config( + config, + vkapi::kFloat, + fp_storage_type, + int8_memory_layout, + /*impl_selector=*/"legacy_4w4c")); + } + } + } + } + + for (Conv2dConfig config : batch_configs) { + const bool is_performance = config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = make_test_case_name( + config, is_performance, utils::kTexture3D, utils::kBuffer); + for (const std::string& pw_selector : pw_selectors) { + test_cases.push_back(create_pw_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4C1W, + pw_selector)); + if (config.batch == 2) { + test_cases.push_back(create_pw_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4W4C, + pw_selector)); + } + } + } + + for (const auto& [edge_config, options] : edge_configs) { + Conv2dConfig config = edge_config; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = + make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); + for (const std::string& pw_selector : pw_selectors) { + test_cases.push_back(create_pw_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4W4C, + pw_selector, + options)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "pw_correctness") { + const std::vector configs = { + // OC < 4 cases to test edge cases with partial output channel blocks + {OutInChannels(1, 16), + InputSize2D(8, 8), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(2, 16), + InputSize2D(8, 8), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(3, 16), + InputSize2D(8, 8), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + // Pointwise convolutions: kernel size 1x1 + {OutInChannels(32, 3), + InputSize2D(64, 64), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(64, 32), + InputSize2D(32, 32), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(96, 64), + InputSize2D(16, 16), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(13, 7), + InputSize2D(57, 33), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(80, 40), + InputSize2D(64, 64), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + }; + const std::vector batch_configs = { + {OutInChannels(8, 8), + InputSize2D(8, 8), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1, + 2}, + }; + const std::vector> + edge_configs = { + {{OutInChannels(13, 7), + InputSize2D(7, 5), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {.input_zero_point = -128, .has_bias = false}}, + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + const std::vector pw_selectors = { + "", "pw_signed", "pw_unsigned", "pw_auto"}; + return { + generate_quantized_conv2d_pw_test_cases( + configs, + batch_configs, + edge_configs, + fp_storage_types, + int8_memory_layouts, + pw_selectors), + pw_reference_impl, + quantized_conv2d_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "pw_performance") { + const std::vector configs = { + // Performance cases (pointwise - will use im2col) + {OutInChannels(160, 480), + InputSize2D(8, 8), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(22, 48), + InputSize2D(256, 256), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(48, 48), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(128, 128), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + {OutInChannels(64, 576), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1}, + }; + const std::vector batch_configs = { + {OutInChannels(64, 32), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1, + 1}, + {OutInChannels(64, 32), + InputSize2D(128, 128), + KernelSize(1, 1), + Stride(1, 1), + Padding(0, 0), + Dilation(1, 1), + 1, + 60}, + }; + const std::vector fp_storage_types = {utils::kTexture3D}; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; + const std::vector pw_selectors = { + "", "pw_signed", "pw_unsigned", "pw_auto"}; + return { + generate_quantized_conv2d_pw_test_cases( + configs, + batch_configs, + /*edge_configs=*/{}, + fp_storage_types, + int8_memory_layouts, + pw_selectors), + pw_reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/scenex_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/scenex_sets.cpp new file mode 100644 index 00000000000..709b6f40c0e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/scenex_sets.cpp @@ -0,0 +1,101 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +std::vector generate_scenex_test_cases( + const std::vector& configs, + const std::vector& routes) { + std::vector test_cases; + for (const std::string& route : routes) { + for (const Conv2dConfig& config : configs) { + test_cases.push_back(create_scenex_test_case(config, route)); + } + } + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "scenex_regular") { + const std::vector configs = { + {OutInChannels(128, 64), + InputSize2D(40, 51), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1, + 60}, + {OutInChannels(256, 128), + InputSize2D(20, 26), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1, + 60}, + }; + const std::vector routes = {"auto", "direct", "im2col"}; + return { + generate_scenex_test_cases(configs, routes), + scenex_direct_reference, + quantized_conv2d_flop_calculator, + /*warmup_runs=*/3, + /*benchmark_runs=*/10}; +} + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "scenex_grouped") { + const std::vector configs = { + {OutInChannels(64, 64), + InputSize2D(128, 128), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2, + 60}, + {OutInChannels(128, 128), + InputSize2D(128, 128), + KernelSize(5, 5), + Stride(2, 2), + Padding(2, 2), + Dilation(1, 1), + 4, + 60}, + {OutInChannels(64, 64), + InputSize2D(64, 64), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2, + 60}, + }; + const std::vector routes = {"auto", "direct", "im2col"}; + return { + generate_scenex_test_cases(configs, routes), + scenex_direct_reference, + quantized_conv2d_flop_calculator, + /*warmup_runs=*/3, + /*benchmark_runs=*/10}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_dynamic_shrink_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_dynamic_shrink_set.cpp new file mode 100644 index 00000000000..c9354b017a6 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_dynamic_shrink_set.cpp @@ -0,0 +1,128 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +void execute_streaming_dynamic_shrink_test() { + Conv2dConfig config = { + OutInChannels(4, 64), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 10}; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = + make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); + TestCase test_case = create_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + utils::kPackedInt8_4W4C, + /*impl_selector=*/"im2col_auto", + /*im2col_options=*/nullptr, + /*input_scale_val=*/1.0f, + /*input_data_gen=*/DataGenType::RANDINT); + for (ValueSpec& input : test_case.inputs()) { + input.ensure_data_generated(/*seed=*/0); + } + + BenchmarkGraph benchmark_graph = setup_compute_graph( + test_case, test_case.operator_name(), /*op_invocations_per_execute=*/1); + ComputeGraph& graph = *benchmark_graph.graph; + graph.prepare(); + graph.prepack(); + + const std::vector upper_input_sizes = test_case.inputs().at(0).sizes; + const std::vector shrunk_input_sizes = {1, 64, 20, 51}; + const std::vector shrunk_output_sizes = {1, 4, 20, 51}; + TestCase shrunk_case = test_case; + shrunk_case.inputs().at(0).sizes = shrunk_input_sizes; + shrunk_case.inputs().at(0).resize_data(1 * 64 * 20 * 51); + shrunk_case.outputs().at(0).sizes = shrunk_output_sizes; + shrunk_case.outputs().at(0).resize_data(1 * 4 * 20 * 51); + reference_impl(shrunk_case); + + constexpr int kRepetitions = 4; + for (int repetition = 0; repetition < kRepetitions; ++repetition) { + graph.resize_input(0, upper_input_sizes); + graph.propagate_resize(); + graph.maybe_cast_and_copy_into_staging( + graph.inputs().at(0).staging, + test_case.inputs().at(0).get_data_ptr(), + test_case.inputs().at(0).numel(), + vkapi::kFloat); + graph.execute(); + + graph.resize_input(0, shrunk_input_sizes); + graph.propagate_resize(); + if (graph.sizes_of(graph.outputs().at(0).value) != shrunk_output_sizes) { + throw std::runtime_error("streaming im2col output did not shrink"); + } + graph.maybe_cast_and_copy_into_staging( + graph.inputs().at(0).staging, + shrunk_case.inputs().at(0).get_data_ptr(), + shrunk_case.inputs().at(0).numel(), + vkapi::kFloat); + graph.execute(); + graph.maybe_cast_and_copy_from_staging( + graph.outputs().at(0).staging, + shrunk_case.outputs().at(0).get_mutable_data_ptr(), + shrunk_case.outputs().at(0).numel(), + vkapi::kFloat); + if (!shrunk_case.outputs().at(0).validate_against_reference( + shrunk_case.get_abs_tolerance(), shrunk_case.get_rel_tolerance())) { + throw std::runtime_error("streaming im2col shrink output was stale"); + } + } + + const Q8taConv2dStreamPlan upper_plan = make_q8ta_conv2d_stream_plan( + /*batch=*/10, + /*flattened_kernel_size=*/576, + /*out_height=*/30, + /*out_width=*/99, + kQ8taConv2dIm2ColScratchBudgetBytes); + if (!upper_plan.feasible || upper_plan.num_tiles != 2 || + upper_plan.rows_per_tile <= 20) { + throw std::runtime_error("dynamic shrink test did not create dead tiles"); + } + const int64_t resized_scratch_bytes = + 576 * upper_plan.rows_per_tile * utils::align_up_4(51); + if (resized_scratch_bytes > kQ8taConv2dIm2ColScratchBudgetBytes) { + throw std::runtime_error("streaming im2col scratch exceeded its cap"); + } + std::cout << "Streaming im2col dynamic shrink PASSED" << std::endl; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "streaming_dynamic_shrink") { + TestCaseSet set; + set.custom_test = execute_streaming_dynamic_shrink_test; + return set; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_im2col_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_im2col_set.cpp new file mode 100644 index 00000000000..b94fbc25140 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/streaming_im2col_set.cpp @@ -0,0 +1,138 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +struct StreamingIm2colConfig { + Conv2dConfig config; + std::vector int8_memory_layouts; + std::string impl_selector; +}; + +std::vector generate_streaming_im2col_test_cases( + const std::vector& configs) { + const bool has_int8_dot_product = + vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product(); + std::vector test_cases; + for (const StreamingIm2colConfig& streaming_config : configs) { + // Forced-fallback cases need no int8 dot-product support, so they run on + // all devices; the other routes stay gated. + if (!has_int8_dot_product && + streaming_config.impl_selector != "im2col_fallback") { + continue; + } + Conv2dConfig config = streaming_config.config; + config.op_name = "conv2d_q8ta_q8csw_q8to"; + config.test_case_name = + make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); + for (const utils::GPUMemoryLayout int8_memory_layout : + streaming_config.int8_memory_layouts) { + test_cases.push_back(create_test_case_from_config( + config, + vkapi::kFloat, + utils::kTexture3D, + int8_memory_layout, + streaming_config.impl_selector, + /*im2col_options=*/nullptr, + /*input_scale_val=*/1.0f, + /*input_data_gen=*/DataGenType::RANDINT)); + } + } + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "streaming_im2col") { + const std::vector configs = { + // Full fit + {{OutInChannels(4, 32), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 10}, + {utils::kPackedInt8_4W4C}, + "im2col_fallback"}, + // Streaming + {{OutInChannels(4, 64), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 10}, + {utils::kPackedInt8_4W4C}, + "im2col_fallback"}, + // Full fit + {{OutInChannels(4, 32), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 10}, + {utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C}, + "im2col_auto"}, + // Streaming + {{OutInChannels(4, 64), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 10}, + {utils::kPackedInt8_4C1W}, + "im2col_auto"}, + // Grouped + {{OutInChannels(16, 32), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 2, + 20}, + {utils::kPackedInt8_4W4C}, + "im2col_auto"}, + // Output channel tail + {{OutInChannels(10, 32), + InputSize2D(30, 99), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1, + 20}, + {utils::kPackedInt8_4W4C}, + "im2col_auto"}, + }; + return { + generate_streaming_im2col_test_cases(configs), + reference_impl, + quantized_conv2d_flop_calculator}; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/unsigned_dot_selection_set.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/unsigned_dot_selection_set.cpp new file mode 100644 index 00000000000..0f568ad30c9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d/test_sets/unsigned_dot_selection_set.cpp @@ -0,0 +1,43 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d { + +namespace { + +void check_unsigned_dot_selection() { + const vkapi::Adapter& adapter = *vkcompute::api::context()->adapter_ptr(); + const bool prefers_unsigned_dot = + adapter.accelerates_unsigned_packed4x8_dot() && + !adapter.accelerates_signed_packed4x8_dot(); + VK_CHECK_COND( + can_use_unsigned_pw_dot(adapter, kMaxUnsignedDotAccumulatorBytes) == + prefers_unsigned_dot); + VK_CHECK_COND( + !can_use_unsigned_pw_dot(adapter, kMaxUnsignedDotAccumulatorBytes + 1)); +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d", "unsigned_dot_selection") { + TestCaseSet set; + set.custom_test = check_unsigned_dot_selection; + return set; +} + +} // namespace q8ta_conv2d +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_transposed.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/conv2d_transposed.cpp similarity index 72% rename from backends/vulkan/test/custom_ops/test_q8ta_conv2d_transposed.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/conv2d_transposed.cpp index b82cb19e614..18c2cd16835 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_conv2d_transposed.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/conv2d_transposed.cpp @@ -4,45 +4,43 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include - -#include -#include - -#include - -#include "conv2d_utils.h" -#include "utils.h" +#include -// #define DEBUG_MODE - -using namespace executorch::vulkan::prototyping; +#include +#include +#include +#include +#include -using namespace vkcompute; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d_transposed { -static constexpr int64_t kRefDimSizeLimit = 100; +namespace { // Transposed convolution output size formula: // H_out = (H_in - 1) * stride_h - 2 * pad_h + dilation_h * (K_h - 1) // + output_pad_h + 1 -static int64_t get_transpose_output_height( +int64_t get_transpose_output_height( const Conv2dConfig& config, int32_t output_pad_h) { return (config.input_size.h - 1) * config.stride.h - 2 * config.padding.h + config.dilation.h * (config.kernel.h - 1) + output_pad_h + 1; } -static int64_t get_transpose_output_width( +int64_t get_transpose_output_width( const Conv2dConfig& config, int32_t output_pad_w) { return (config.input_size.w - 1) * config.stride.w - 2 * config.padding.w + config.dilation.w * (config.kernel.w - 1) + output_pad_w + 1; } +} // namespace + // Utility function to create a test case from a Conv2dConfig for transposed // convolution -static TestCase create_test_case_from_config( +TestCase create_test_case_from_config( const Conv2dConfig& config, int32_t output_pad_h, int32_t output_pad_w, @@ -209,163 +207,31 @@ static TestCase create_test_case_from_config( return test_case; } -// Generate easy test cases for debugging -std::vector generate_quantized_conv2d_transposed_easy_cases() { - std::vector test_cases; - - Conv2dConfig config = { - OutInChannels(16, 32), - InputSize2D(8, 8), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1, - }; - - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - for (const utils::GPUMemoryLayout int8_memory_layout : int8_memory_layouts) { - config.test_case_name = - make_test_case_name(config, false, utils::kTexture3D, utils::kBuffer); - test_cases.push_back(create_test_case_from_config( - config, - /*output_pad_h=*/1, - /*output_pad_w=*/1, - vkapi::kFloat, - utils::kTexture3D, - int8_memory_layout)); - } +int64_t quantized_conv2d_transposed_flop_calculator(const TestCase& test_case) { + int kernel_idx = 9; - return test_cases; -} + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); + const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); -// Generate test cases for quantized transposed conv2d -static std::vector generate_quantized_conv2d_transposed_test_cases() { - std::vector test_cases; - if (!vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { - return test_cases; - } + int64_t N = input_sizes[0]; + int64_t C_in = input_sizes[1]; + int64_t C_out = output_sizes[1]; + int64_t K_h = kernel_sizes[0]; + int64_t K_w = kernel_sizes[1]; + int64_t H_out = output_sizes[2]; + int64_t W_out = output_sizes[3]; - // Each entry: {config, output_pad_h, output_pad_w} - struct TransposedConvTestConfig { - Conv2dConfig config; - int32_t output_pad_h; - int32_t output_pad_w; - }; - - std::vector configs = { - // Basic transposed conv (stride=2, common in decoder networks) - {{OutInChannels(16, 32), - InputSize2D(8, 8), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - 1, - 1}, - {{OutInChannels(32, 64), - InputSize2D(4, 4), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - 1, - 1}, - // No output padding - {{OutInChannels(16, 32), - InputSize2D(8, 8), - KernelSize(4, 4), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - 0, - 0}, - // Stride=1 (degenerate case) - {{OutInChannels(16, 16), - InputSize2D(8, 8), - KernelSize(3, 3), - Stride(1, 1), - Padding(1, 1), - Dilation(1, 1), - 1}, - 0, - 0}, - // Grouped transposed conv - {{OutInChannels(32, 64), - InputSize2D(8, 8), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 2}, - 1, - 1}, - // Larger spatial - {{OutInChannels(64, 128), - InputSize2D(16, 16), - KernelSize(4, 4), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - 0, - 0}, - // Performance cases - {{OutInChannels(64, 128), - InputSize2D(32, 32), - KernelSize(3, 3), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - 1, - 1}, - {{OutInChannels(128, 256), - InputSize2D(16, 16), - KernelSize(4, 4), - Stride(2, 2), - Padding(1, 1), - Dilation(1, 1), - 1}, - 0, - 0}, - }; - - std::vector int8_memory_layouts = { - utils::kPackedInt8_4C1W, utils::kPackedInt8_4W4C, utils::kPackedInt8_4C}; - - for (auto& tc : configs) { - auto& config = tc.config; - bool is_performance = config.channels.out > kRefDimSizeLimit || - config.channels.in > kRefDimSizeLimit || - config.input_size.h > kRefDimSizeLimit || - config.input_size.w > kRefDimSizeLimit; - - for (const utils::GPUMemoryLayout int8_memory_layout : - int8_memory_layouts) { - config.test_case_name = make_test_case_name( - config, is_performance, utils::kTexture3D, utils::kBuffer); - - test_cases.push_back(create_test_case_from_config( - config, - tc.output_pad_h, - tc.output_pad_w, - vkapi::kFloat, - utils::kTexture3D, - int8_memory_layout)); - } - } + int64_t output_elements = N * C_out * H_out * W_out; + int64_t ops_per_output = C_in * K_h * K_w; - return test_cases; + return output_elements * ops_per_output; } +namespace { + // Reference implementation for quantized transposed conv2d -static void conv2d_transposed_q8ta_reference_impl(TestCase& test_case) { +void conv2d_transposed_q8ta_reference_impl(TestCase& test_case) { int32_t idx = 0; const ValueSpec& input_spec = test_case.inputs()[idx++]; const ValueSpec& input_scale_spec = test_case.inputs()[idx++]; @@ -544,61 +410,13 @@ static void conv2d_transposed_q8ta_reference_impl(TestCase& test_case) { } } -static void reference_impl(TestCase& test_case) { - conv2d_transposed_q8ta_reference_impl(test_case); -} - -static int64_t quantized_conv2d_transposed_flop_calculator( - const TestCase& test_case) { - int kernel_idx = 9; - - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& output_sizes = test_case.outputs()[0].get_tensor_sizes(); - const auto& kernel_sizes = test_case.inputs()[kernel_idx].get_int32_data(); - - int64_t N = input_sizes[0]; - int64_t C_in = input_sizes[1]; - int64_t C_out = output_sizes[1]; - int64_t K_h = kernel_sizes[0]; - int64_t K_w = kernel_sizes[1]; - int64_t H_out = output_sizes[2]; - int64_t W_out = output_sizes[3]; - - int64_t output_elements = N * C_out * H_out * W_out; - int64_t ops_per_output = C_in * K_h * K_w; +} // namespace - return output_elements * ops_per_output; +void reference_impl(TestCase& test_case) { + conv2d_transposed_q8ta_reference_impl(test_case); } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(true); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout - << "Quantized Transposed Conv2d Operation with Output Quantization Prototyping Framework" - << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( -#ifdef DEBUG_MODE - generate_quantized_conv2d_transposed_easy_cases, -#else - generate_quantized_conv2d_transposed_test_cases, -#endif - quantized_conv2d_transposed_flop_calculator, - "QuantizedTransposedConv2d", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_conv2d_transposed +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/conv2d_transposed.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/conv2d_transposed.h new file mode 100644 index 00000000000..34873570947 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/conv2d_transposed.h @@ -0,0 +1,35 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d_transposed { + +constexpr int64_t kRefDimSizeLimit = 100; + +TestCase create_test_case_from_config( + const Conv2dConfig& config, + int32_t output_pad_h, + int32_t output_pad_w, + vkapi::ScalarType input_dtype, + utils::StorageType fp_storage_type, + utils::GPUMemoryLayout int8_memory_layout); + +void reference_impl(TestCase& test_case); + +int64_t quantized_conv2d_transposed_flop_calculator(const TestCase& test_case); + +} // namespace q8ta_conv2d_transposed +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/test_sets/conv2d_transposed_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/test_sets/conv2d_transposed_sets.cpp new file mode 100644 index 00000000000..46bdef86bcb --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/conv2d_transposed/test_sets/conv2d_transposed_sets.cpp @@ -0,0 +1,196 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_conv2d_transposed { + +namespace { + +// Each entry: {config, output_pad_h, output_pad_w} +struct TransposedConvTestConfig { + Conv2dConfig config; + int32_t output_pad_h; + int32_t output_pad_w; +}; + +std::vector generate_test_cases( + const std::vector& configs, + const std::vector& int8_memory_layouts, + bool require_int8_dot_product = true) { + std::vector test_cases; + if (require_int8_dot_product && + !vkcompute::api::context()->adapter_ptr()->supports_int8_dot_product()) { + return test_cases; + } + + for (auto tc : configs) { + auto& config = tc.config; + bool is_performance = config.channels.out > kRefDimSizeLimit || + config.channels.in > kRefDimSizeLimit || + config.input_size.h > kRefDimSizeLimit || + config.input_size.w > kRefDimSizeLimit; + + for (const utils::GPUMemoryLayout int8_memory_layout : + int8_memory_layouts) { + config.test_case_name = make_test_case_name( + config, is_performance, utils::kTexture3D, utils::kBuffer); + + test_cases.push_back(create_test_case_from_config( + config, + tc.output_pad_h, + tc.output_pad_w, + vkapi::kFloat, + utils::kTexture3D, + int8_memory_layout)); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/conv2d_transposed", "correctness") { + const std::vector configs = { + // Basic transposed conv (stride=2, common in decoder networks) + {{OutInChannels(16, 32), + InputSize2D(8, 8), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + 1, + 1}, + {{OutInChannels(32, 64), + InputSize2D(4, 4), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + 1, + 1}, + // No output padding + {{OutInChannels(16, 32), + InputSize2D(8, 8), + KernelSize(4, 4), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + 0, + 0}, + // Stride=1 (degenerate case) + {{OutInChannels(16, 16), + InputSize2D(8, 8), + KernelSize(3, 3), + Stride(1, 1), + Padding(1, 1), + Dilation(1, 1), + 1}, + 0, + 0}, + // Grouped transposed conv + {{OutInChannels(32, 64), + InputSize2D(8, 8), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 2}, + 1, + 1}, + }; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4C, + }; + return { + generate_test_cases(configs, int8_memory_layouts), + reference_impl, + quantized_conv2d_transposed_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8ta/conv2d_transposed", "debug") { + const std::vector configs = { + // Easy test case for debugging + {{OutInChannels(16, 32), + InputSize2D(8, 8), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + /*output_pad_h=*/1, + /*output_pad_w=*/1}, + }; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4C, + }; + return { + generate_test_cases( + configs, int8_memory_layouts, /*require_int8_dot_product=*/false), + reference_impl, + quantized_conv2d_transposed_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8ta/conv2d_transposed", "performance") { + const std::vector configs = { + // Larger spatial + {{OutInChannels(64, 128), + InputSize2D(16, 16), + KernelSize(4, 4), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + 0, + 0}, + // Performance cases + {{OutInChannels(64, 128), + InputSize2D(32, 32), + KernelSize(3, 3), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + 1, + 1}, + {{OutInChannels(128, 256), + InputSize2D(16, 16), + KernelSize(4, 4), + Stride(2, 2), + Padding(1, 1), + Dilation(1, 1), + 1}, + 0, + 0}, + }; + const std::vector int8_memory_layouts = { + utils::kPackedInt8_4C1W, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4C, + }; + return { + generate_test_cases(configs, int8_memory_layouts), + reference_impl, + quantized_conv2d_transposed_flop_calculator}; +} + +} // namespace q8ta_conv2d_transposed +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_q8ta_linear.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/linear.cpp similarity index 74% rename from backends/vulkan/test/custom_ops/test_q8ta_linear.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/linear/linear.cpp index 2b65b53c896..3629d293eb0 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_linear.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/linear.cpp @@ -4,34 +4,23 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include - -#include -#include - -#include - -#include "utils.h" - -using namespace executorch::vulkan::prototyping; - -using namespace vkcompute; +#include -static constexpr int64_t kRefDimSizeLimit = 300; +#include +#include +#include +#include +#include -struct LinearConfig { - int64_t M; - int64_t K; - int64_t N; - bool has_bias = true; - std::string test_case_name = "placeholder"; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_linear { -static TestCase create_test_case_from_config( +TestCase create_test_case_from_config( const LinearConfig& config, vkapi::ScalarType input_dtype, - const std::string& impl_selector = "") { + const std::string& impl_selector) { TestCase test_case; bool is_perf = config.M >= kRefDimSizeLimit || config.K >= kRefDimSizeLimit || @@ -172,63 +161,26 @@ static TestCase create_test_case_from_config( return test_case; } -// Generate test cases for q8ta_linear operation -static std::vector generate_q8ta_linear_test_cases() { - std::vector test_cases; - - std::vector configs = { - // Batch size 1 cases (test both tiled and gemv) - {1, 64, 32}, - {1, 128, 64}, - {1, 256, 128}, - {1, 128, 64, false}, - // Multi-batch cases - {4, 64, 32}, - {4, 128, 64}, - {4, 256, 128}, - {32, 64, 32}, - {32, 128, 64}, - {32, 256, 128}, - // No bias tests - {32, 128, 64, false}, - {32, 256, 128, false}, - // Performance cases - {1, 512, 512}, - {1, 2048, 2048}, - {1, 512, 9059}, - {256, 2048, 2048}, - {512, 2048, 2048}, - {1024, 2048, 2048}, - }; - - for (auto config : configs) { - bool is_performance = config.M >= kRefDimSizeLimit || - config.K >= kRefDimSizeLimit || config.N >= kRefDimSizeLimit; - - std::string prefix = is_performance ? "performance_" : "correctness_"; - std::string generated_test_case_name = prefix + std::to_string(config.M) + - "_" + std::to_string(config.K) + "_" + std::to_string(config.N); - if (!config.has_bias) { - generated_test_case_name += "_no_bias"; - } +int64_t q8ta_linear_flop_calculator(const TestCase& test_case) { + const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto& weight_sizes = test_case.inputs()[3].get_tensor_sizes(); - config.test_case_name = generated_test_case_name; + int64_t batch_size = input_sizes[0]; + int64_t in_features = input_sizes[1]; + int64_t out_features = weight_sizes[0]; - // Default (tiled) variant - test_cases.push_back(create_test_case_from_config(config, vkapi::kFloat)); + int64_t output_elements = batch_size * out_features; + int64_t ops_per_output = in_features; - // For batch size 1, also test the gemv variant - if (config.M == 1) { - test_cases.push_back( - create_test_case_from_config(config, vkapi::kFloat, "gemv")); - } - } + int64_t flop = output_elements * ops_per_output; - return test_cases; + return flop; } +namespace { + // Reference implementation for q8ta_linear (activation+weight+output quantized) -static void q8ta_linear_reference_impl(TestCase& test_case) { +void q8ta_linear_reference_impl(TestCase& test_case) { int32_t idx = 0; const ValueSpec& input_spec = test_case.inputs()[idx++]; const ValueSpec& input_scale_spec = test_case.inputs()[idx++]; @@ -329,45 +281,13 @@ static void q8ta_linear_reference_impl(TestCase& test_case) { } } -static void reference_impl(TestCase& test_case) { - q8ta_linear_reference_impl(test_case); -} - -static int64_t q8ta_linear_flop_calculator(const TestCase& test_case) { - const auto& input_sizes = test_case.inputs()[0].get_tensor_sizes(); - const auto& weight_sizes = test_case.inputs()[3].get_tensor_sizes(); - - int64_t batch_size = input_sizes[0]; - int64_t in_features = input_sizes[1]; - int64_t out_features = weight_sizes[0]; - - int64_t output_elements = batch_size * out_features; - int64_t ops_per_output = in_features; - - int64_t flop = output_elements * ops_per_output; +} // namespace - return flop; +void reference_impl(TestCase& test_case) { + q8ta_linear_reference_impl(test_case); } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Q8ta Linear Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = reference_impl; - - auto results = execute_test_cases( - generate_q8ta_linear_test_cases, - q8ta_linear_flop_calculator, - "Q8taLinear", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/linear.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/linear.h new file mode 100644 index 00000000000..0f27844ff43 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/linear.h @@ -0,0 +1,41 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_linear { + +constexpr int64_t kRefDimSizeLimit = 300; + +struct LinearConfig { + int64_t M; + int64_t K; + int64_t N; + bool has_bias = true; + std::string test_case_name = "placeholder"; +}; + +TestCase create_test_case_from_config( + const LinearConfig& config, + vkapi::ScalarType input_dtype, + const std::string& impl_selector = ""); + +void reference_impl(TestCase& test_case); + +int64_t q8ta_linear_flop_calculator(const TestCase& test_case); + +} // namespace q8ta_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/test_sets/linear_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/test_sets/linear_sets.cpp new file mode 100644 index 00000000000..1483728fc47 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/linear/test_sets/linear_sets.cpp @@ -0,0 +1,96 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_linear { + +namespace { + +std::vector generate_test_cases( + const std::vector& configs) { + std::vector test_cases; + + for (auto config : configs) { + bool is_performance = config.M >= kRefDimSizeLimit || + config.K >= kRefDimSizeLimit || config.N >= kRefDimSizeLimit; + + std::string prefix = is_performance ? "performance_" : "correctness_"; + std::string generated_test_case_name = prefix + std::to_string(config.M) + + "_" + std::to_string(config.K) + "_" + std::to_string(config.N); + if (!config.has_bias) { + generated_test_case_name += "_no_bias"; + } + + config.test_case_name = generated_test_case_name; + + // Default (tiled) variant + test_cases.push_back(create_test_case_from_config(config, vkapi::kFloat)); + + // For batch size 1, also test the gemv variant + if (config.M == 1) { + test_cases.push_back( + create_test_case_from_config(config, vkapi::kFloat, "gemv")); + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/linear", "correctness") { + const std::vector configs = { + // Batch size 1 cases (test both tiled and gemv) + {1, 64, 32}, + {1, 128, 64}, + {1, 256, 128}, + {1, 128, 64, false}, + // Multi-batch cases + {4, 64, 32}, + {4, 128, 64}, + {4, 256, 128}, + {32, 64, 32}, + {32, 128, 64}, + {32, 256, 128}, + // No bias tests + {32, 128, 64, false}, + {32, 256, 128, false}, + }; + + return { + generate_test_cases(configs), + reference_impl, + q8ta_linear_flop_calculator}; +} + +REGISTER_TEST_CASE_SET("q8ta/linear", "performance") { + const std::vector configs = { + // Performance cases + {1, 512, 512}, + {1, 2048, 2048}, + {1, 512, 9059}, + {256, 2048, 2048}, + {512, 2048, 2048}, + {1024, 2048, 2048}, + }; + + return { + generate_test_cases(configs), + reference_impl, + q8ta_linear_flop_calculator}; +} + +} // namespace q8ta_linear +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_q8ta_pixel_shuffle.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/pixel_shuffle.cpp similarity index 57% rename from backends/vulkan/test/custom_ops/test_q8ta_pixel_shuffle.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/pixel_shuffle.cpp index b932d1e92d3..5f805171fac 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_pixel_shuffle.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/pixel_shuffle.cpp @@ -4,41 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include +#include + #include #include -#include -#include +#include +#include #include -#include "utils.h" - -#include - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 512; -// Test op signature in TestQ8taPixelShuffle.cpp: -// test_q8ta_pixel_shuffle(fp_in, in_scale, in_zp, out_scale, out_zp, -// upscale_factor, in_layout, out_layout) -> fp_out -// Implementation: fused fast-path kernel. The in_layout / out_layout strings -// select the channels-packed int8x4 layout used for the temporary quantized -// tensors. Supported values: "4W4C", "4C1W". -// (PACKED_INT8_CONV2D is a Python/serialization-level alias that the runtime -// resolves to kPackedInt8_4C1W, so it is not exercised separately here -- it -// would only re-test the same C++ kernel path as "4C1W".) - -struct PixelShuffleConfig { - std::vector in_shape; // [N, C*r*r, H, W] - int upscale_factor; - bool same_qparams; // if true, in_scale == out_scale and in_zp == out_zp - std::string in_layout = "4W4C"; - std::string out_layout = "4W4C"; - std::string test_case_name = "ACCU"; - std::string op_name = "test_q8ta_pixel_shuffle"; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_pixel_shuffle { TestCase create_test_case_from_config(const PixelShuffleConfig& config) { TestCase test_case; @@ -124,92 +101,6 @@ TestCase create_test_case_from_config(const PixelShuffleConfig& config) { return test_case; } -// All (in_layout, out_layout) pairs across the channels-packed int8 family. -// CONV2D is a Python-level alias that resolves to 4C1W at the C++ runtime, so -// it is not listed separately -- it would just re-run the 4C1W kernel path. -static const std::vector>& -get_layout_pairs() { - static const std::vector> layout_pairs = { - {"4W4C", "4W4C"}, - {"4W4C", "4C1W"}, - {"4C1W", "4W4C"}, - {"4C1W", "4C1W"}, - }; - return layout_pairs; -} - -std::vector generate_correctness_cases() { - std::vector test_cases; - - // Small shapes, all use r=2 (the only factor needed by the model). - // Shape format is the *input* shape [N, C_in, H, W] where C_in = C_out * r*r. - std::vector> shapes = { - // Small even W to be a multiple of 4 after upscaling. - {1, 16, 4, 4}, // out: [1, 4, 8, 8] - {1, 24, 8, 4}, // out: [1, 6, 16, 8] - {1, 32, 12, 8}, // out: [1, 8, 24, 16] - {1, 96, 16, 9}, // out: [1, 24, 32, 18] - first model shape - }; - - for (const auto& shape : shapes) { - for (bool same_qp : {true, false}) { - for (const auto& layouts : get_layout_pairs()) { - PixelShuffleConfig cfg; - cfg.in_shape = shape; - cfg.upscale_factor = 2; - cfg.same_qparams = same_qp; - cfg.in_layout = layouts.first; - cfg.out_layout = layouts.second; - cfg.test_case_name = "ACCU"; - - test_cases.push_back(create_test_case_from_config(cfg)); - } - } - } - - return test_cases; -} - -std::vector generate_perf_cases() { - std::vector test_cases; - - // Model perf shapes (output shapes from the RefineNet decoder; we compute - // the input shape as [N, C_out * r*r, H_out / r, W_out / r]). - // Output shapes: [1, 24, 32, 18], [1, 24, 64, 36], [1, 24, 128, 72], - // [1, 24, 256, 144]. For r=2, in shapes = [1, 96, 16, 9], etc. - std::vector> in_shapes = { - {1, 96, 16, 9}, - {1, 96, 32, 18}, - {1, 96, 64, 36}, - {1, 96, 128, 72}, - }; - - for (const auto& shape : in_shapes) { - for (const auto& layouts : get_layout_pairs()) { - PixelShuffleConfig cfg; - cfg.in_shape = shape; - cfg.upscale_factor = 2; - cfg.same_qparams = true; // residual-style: scales match - cfg.in_layout = layouts.first; - cfg.out_layout = layouts.second; - cfg.test_case_name = "PERF"; - - test_cases.push_back(create_test_case_from_config(cfg)); - } - } - - return test_cases; -} - -std::vector generate_all_cases() { - std::vector all = generate_correctness_cases(); - std::vector perf = generate_perf_cases(); - for (auto& tc : perf) { - all.push_back(tc); - } - return all; -} - // Reference: quantize input, do PyTorch-equivalent pixel shuffle, requantize, // then dequantize for comparison. void q8ta_pixel_shuffle_reference_impl(TestCase& test_case) { @@ -288,24 +179,7 @@ void q8ta_pixel_shuffle_reference_impl(TestCase& test_case) { } } -int main(int /*argc*/, char* /*argv*/[]) { - set_debugging(false); - set_print_output(false); - set_print_latencies(false); - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Q8TA PixelShuffle Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = q8ta_pixel_shuffle_reference_impl; - - auto results = execute_test_cases( - generate_all_cases, - "Q8taPixelShuffle", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_pixel_shuffle +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/pixel_shuffle.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/pixel_shuffle.h new file mode 100644 index 00000000000..42dcc329359 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/pixel_shuffle.h @@ -0,0 +1,49 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_pixel_shuffle { + +constexpr int64_t kRefDimSizeLimit = 512; + +// Test op signature in TestQ8taPixelShuffle.cpp: +// test_q8ta_pixel_shuffle(fp_in, in_scale, in_zp, out_scale, out_zp, +// upscale_factor, in_layout, out_layout) -> fp_out +// Implementation: fused fast-path kernel. The in_layout / out_layout strings +// select the channels-packed int8x4 layout used for the temporary quantized +// tensors. Supported values: "4W4C", "4C1W". +// (PACKED_INT8_CONV2D is a Python/serialization-level alias that the runtime +// resolves to kPackedInt8_4C1W, so it is not exercised separately here -- it +// would only re-test the same C++ kernel path as "4C1W".) + +struct PixelShuffleConfig { + std::vector in_shape; // [N, C*r*r, H, W] + int upscale_factor; + bool same_qparams; // if true, in_scale == out_scale and in_zp == out_zp + std::string in_layout = "4W4C"; + std::string out_layout = "4W4C"; + std::string test_case_name = "ACCU"; + std::string op_name = "test_q8ta_pixel_shuffle"; +}; + +TestCase create_test_case_from_config(const PixelShuffleConfig& config); + +void q8ta_pixel_shuffle_reference_impl(TestCase& test_case); + +} // namespace q8ta_pixel_shuffle +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/test_sets/pixel_shuffle_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/test_sets/pixel_shuffle_sets.cpp new file mode 100644 index 00000000000..13605714009 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/pixel_shuffle/test_sets/pixel_shuffle_sets.cpp @@ -0,0 +1,96 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_pixel_shuffle { + +namespace { + +// All (in_layout, out_layout) pairs across the channels-packed int8 family. +// CONV2D is a Python-level alias that resolves to 4C1W at the C++ runtime, so +// it is not listed separately -- it would just re-run the 4C1W kernel path. +const std::vector> kLayoutPairs = { + {"4W4C", "4W4C"}, + {"4W4C", "4C1W"}, + {"4C1W", "4W4C"}, + {"4C1W", "4C1W"}, +}; + +std::vector generate_test_cases( + const std::vector>& in_shapes, + int upscale_factor, + const std::vector& same_qparams, + const std::string& test_case_name) { + std::vector test_cases; + + for (const auto& shape : in_shapes) { + for (bool same_qp : same_qparams) { + for (const auto& layouts : kLayoutPairs) { + PixelShuffleConfig cfg; + cfg.in_shape = shape; + cfg.upscale_factor = upscale_factor; + cfg.same_qparams = same_qp; + cfg.in_layout = layouts.first; + cfg.out_layout = layouts.second; + cfg.test_case_name = test_case_name; + + test_cases.push_back(create_test_case_from_config(cfg)); + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/pixel_shuffle", "correctness") { + // Small shapes, all use r=2 (the only factor needed by the model). + // Shape format is the *input* shape [N, C_in, H, W] where C_in = C_out * r*r. + const std::vector> in_shapes = { + // Small even W to be a multiple of 4 after upscaling. + {1, 16, 4, 4}, // out: [1, 4, 8, 8] + {1, 24, 8, 4}, // out: [1, 6, 16, 8] + {1, 32, 12, 8}, // out: [1, 8, 24, 16] + {1, 96, 16, 9}, // out: [1, 24, 32, 18] - first model shape + }; + const std::vector same_qparams = {true, false}; + return { + generate_test_cases( + in_shapes, /*upscale_factor=*/2, same_qparams, "ACCU"), + q8ta_pixel_shuffle_reference_impl}; +} + +REGISTER_TEST_CASE_SET("q8ta/pixel_shuffle", "performance") { + // Model perf shapes (output shapes from the RefineNet decoder; we compute + // the input shape as [N, C_out * r*r, H_out / r, W_out / r]). + // Output shapes: [1, 24, 32, 18], [1, 24, 64, 36], [1, 24, 128, 72], + // [1, 24, 256, 144]. For r=2, in shapes = [1, 96, 16, 9], etc. + const std::vector> in_shapes = { + {1, 96, 16, 9}, + {1, 96, 32, 18}, + {1, 96, 64, 36}, + {1, 96, 128, 72}, + }; + const std::vector same_qparams = {true}; // residual-style: scales match + return { + generate_test_cases( + in_shapes, /*upscale_factor=*/2, same_qparams, "PERF"), + q8ta_pixel_shuffle_reference_impl}; +} + +} // namespace q8ta_pixel_shuffle +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.cpp new file mode 100644 index 00000000000..b6ad788b9db --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.cpp @@ -0,0 +1,164 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_qdq { + +// Utility function to create a test case from a QDQ8BitConfig +TestCase create_test_case_from_config( + const QDQ8BitConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype, + utils::GPUMemoryLayout fp_memory_layout, + utils::GPUMemoryLayout quantized_memory_layout, + const std::string& impl_selector) { + TestCase test_case; + + // Create a descriptive name for the test case + // QDQ: fp input -> int8 staging -> fp output (label as f32->f32) + std::string prefix = config.test_case_name; // "ACCU" or "PERF" + std::string storage_str = repr_str(storage_type, fp_memory_layout) + "->" + + repr_str(utils::kBuffer, quantized_memory_layout); + std::string suffix = impl_selector.empty() ? "" : "[" + impl_selector + "]"; + std::string test_name = make_test_label( + prefix, + dtype_short(input_dtype), + dtype_short(input_dtype), + shape_bracket(config.shape), + storage_str, + suffix); + test_case.set_name(test_name); + + // Set the operator name for the test case + std::string operator_name = "test_etvk." + config.op_name + ".default"; + test_case.set_operator_name(operator_name); + + // Input tensor (float) - any dimensionality + ValueSpec input_tensor( + config.shape, + input_dtype, + storage_type, + fp_memory_layout, + DataGenType::RANDOM); + + float scale_val = 0.007112; + ValueSpec scale(scale_val); + + // Generate random zero point within quantization range + int32_t zero_point_val = 0; + ValueSpec zero_point(zero_point_val); + + // GPUMemoryLayout as integer (will be cast in the operator) + int32_t layout_int = static_cast(quantized_memory_layout); + ValueSpec layout_spec(layout_int); + + // impl_selector string + ValueSpec impl_selector_spec = ValueSpec::make_string(impl_selector); + + // Output tensor (float) - same shape as input + ValueSpec output_tensor( + config.shape, + input_dtype, + storage_type, + fp_memory_layout, + DataGenType::ZEROS); + + // Add all specs to test case + test_case.add_input_spec(input_tensor); + test_case.add_input_spec(scale); + test_case.add_input_spec(zero_point); + test_case.add_input_spec(layout_spec); + test_case.add_input_spec(impl_selector_spec); + test_case.add_output_spec(output_tensor); + + test_case.set_abs_tolerance(scale_val + 1e-4); + + // Use layout-only filter for this test since quantize/dequantize ARE the + // operations being tested, not overhead + test_case.set_shader_filter(kLayoutOnlyShaderFilter); + + return test_case; +} + +// Reference implementation for q_dq_8bit operation +void q_dq_8bit_reference_impl(TestCase& test_case) { + int32_t idx = 0; + const ValueSpec& input_spec = test_case.inputs()[idx++]; + const ValueSpec& scale_spec = test_case.inputs()[idx++]; + const ValueSpec& zero_point_spec = test_case.inputs()[idx++]; + const ValueSpec& layout_spec = test_case.inputs()[idx++]; + (void)layout_spec; // Not used in reference implementation + + // Extract output specification + ValueSpec& output_spec = test_case.outputs()[0]; + + // Get tensor dimensions (arbitrary dimensionality) + auto input_sizes = input_spec.get_tensor_sizes(); + + // Calculate total number of elements + int64_t num_elements = 1; + for (const auto& dim : input_sizes) { + num_elements *= dim; + } + + // Skip for large tensors since computation time will be extremely slow + for (const auto& dim : input_sizes) { + if (dim > kRefDimSizeLimit) { + throw std::invalid_argument( + "One or more dimensions exceed the allowed limit for reference implementation."); + } + } + + if (input_spec.dtype != vkapi::kFloat) { + throw std::invalid_argument("Unsupported dtype"); + } + + // Get raw data pointers + auto& input_data = input_spec.get_float_data(); + + // Extract the randomized scale and zero point values + float scale = scale_spec.get_float_value(); + int32_t zero_point = zero_point_spec.get_int_value(); + int32_t quant_min = -128; + int32_t quant_max = 127; + + // Prepare output data + auto& ref_data = output_spec.get_ref_float_data(); + ref_data.resize(num_elements); + + // Perform quantize-dequantize operation on each element + for (int64_t i = 0; i < num_elements; ++i) { + float input_val = input_data[i]; + + // Quantize: quantized = round(input / scale + zero_point) + float quantized_float = std::round(input_val / scale) + zero_point; + + // Clamp to quantization range + quantized_float = std::max(quantized_float, static_cast(quant_min)); + quantized_float = std::min(quantized_float, static_cast(quant_max)); + + int32_t quantized_int = static_cast(quantized_float); + + // Dequantize: output = (quantized - zero_point) * scale + float dequantized = (quantized_int - zero_point) * scale; + + ref_data[i] = dequantized; + } +} + +} // namespace q8ta_qdq +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.h new file mode 100644 index 00000000000..7b63610edff --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/qdq.h @@ -0,0 +1,42 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_qdq { + +constexpr int64_t kRefDimSizeLimit = 512; + +// Configuration struct for tensor quantize-dequantize testing +struct QDQ8BitConfig { + std::vector shape; // Tensor shape (can be any dimensionality) + std::string test_case_name = "placeholder"; + std::string op_name = "q_dq_8bit_per_tensor"; +}; + +TestCase create_test_case_from_config( + const QDQ8BitConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype, + utils::GPUMemoryLayout fp_memory_layout, + utils::GPUMemoryLayout quantized_memory_layout, + const std::string& impl_selector = ""); + +void q_dq_8bit_reference_impl(TestCase& test_case); + +} // namespace q8ta_qdq +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_sets/qdq_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_sets/qdq_sets.cpp new file mode 100644 index 00000000000..18aa2cce638 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/qdq/test_sets/qdq_sets.cpp @@ -0,0 +1,202 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_qdq { + +namespace { + +std::vector generate_test_cases( + const std::vector>& shapes, + const std::vector& fp_layouts, + const std::vector& quant_layouts, + const std::vector& storage_types, + const std::vector& legacy_4w4c_fp_layouts) { + std::vector test_cases; + + // Generate all combinations + for (const auto& shape : shapes) { + // Generate test case name prefix from shape dimensions + std::string prefix = "ACCU"; + for (const auto& dim : shape) { + if (dim > kRefDimSizeLimit) { + prefix = "PERF"; + break; + } + } + + const bool is_highdim = shape.size() > 4; + + for (const auto& fp_layout : fp_layouts) { + const bool test_legacy_4w4c = + std::find( + legacy_4w4c_fp_layouts.begin(), + legacy_4w4c_fp_layouts.end(), + fp_layout) != legacy_4w4c_fp_layouts.end(); + for (const auto& quant_layout : quant_layouts) { + for (const auto& storage_type : storage_types) { + // Textures are limited to 4D + if (is_highdim && storage_type != utils::kBuffer) { + continue; + } + QDQ8BitConfig config; + config.shape = shape; + config.test_case_name = prefix; + + test_cases.push_back(create_test_case_from_config( + config, storage_type, vkapi::kFloat, fp_layout, quant_layout)); + // For 4W4C layout, also test with legacy implementation + // (legacy path doesn't support high-dim tensors) + if (!is_highdim && test_legacy_4w4c && + quant_layout == utils::kPackedInt8_4W4C) { + test_cases.push_back(create_test_case_from_config( + config, + storage_type, + vkapi::kFloat, + fp_layout, + quant_layout, + /*impl_selector=*/"legacy_4w4c")); + } + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/qdq", "correctness") { + // Shapes to test (no layout specified - will be combined with all + // layouts) + const std::vector> shapes = { + // 1D tensors + {144}, + {90}, + + // 2D tensors (exercises block config with ndim < 4) + {1, 144}, + {1, 90}, + {1, 4}, + {3, 32}, + + // 3D tensors + {1, 16, 32}, + {1, 3, 64}, + + // Small test cases for correctness + {1, 3, 16, 16}, + {1, 8, 32, 32}, + {1, 16, 24, 24}, + {1, 32, 12, 12}, + {1, 1, 64, 64}, + {1, 3, 64, 64}, + {1, 4, 16, 16}, + + // Different tensor sizes + {1, 8, 20, 20}, + {1, 16, 14, 14}, + {1, 8, 28, 28}, + + // Odd tensor sizes + {1, 3, 15, 15}, + {1, 13, 31, 31}, + {1, 17, 23, 23}, + + // Performance test cases (larger tensors) + {1, 64, 128, 128}, + {1, 32, 64, 64}, + {1, 128, 56, 56}, + + // 5D tensors (high-dim batch support) + {2, 1, 3, 16, 16}, + {1, 2, 8, 8, 8}, + + // 6D tensors (high-dim batch support) + {2, 1, 1, 4, 8, 16}, + {1, 1, 2, 3, 8, 8}, + }; + // FP memory layouts to test + const std::vector fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + // Quantized memory layouts to test + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + // Test with buffer storage only - the unified block-based shaders only + // support buffer-backed floating-point tensors. Texture storage is tested + // separately by qdq8ta_conv2d_activations which uses the layout-specific + // shaders. + const std::vector storage_types = { + utils::kBuffer, utils::kTexture3D}; + const std::vector legacy_4w4c_fp_layouts = { + utils::kChannelsPacked}; + return { + generate_test_cases( + shapes, + fp_layouts, + quant_layouts, + storage_types, + legacy_4w4c_fp_layouts), + q_dq_8bit_reference_impl}; +} + +// Easy test cases for q_dq_8bit operation (for debugging) +REGISTER_TEST_CASE_SET("q8ta/qdq", "debug") { + const std::vector> shapes = { + {1, 16, 16, 16}, // 4D: [N, C, H, W] + {1, 144}, // 2D: exercises block config with ndim < 4 + {1, 90}, // 2D: matches skin_seg model's keypoint/bbox tensor sizes + {2, 1, 3, 16, 16}, // 5D: exercises high-dim batch support + {2, 1, 1, 4, 8, 16}, // 6D: exercises high-dim batch support + }; + // FP memory layouts to test + const std::vector fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + // Quantized memory layouts to test + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + const std::vector storage_types = {utils::kBuffer}; + const std::vector legacy_4w4c_fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + return { + generate_test_cases( + shapes, + fp_layouts, + quant_layouts, + storage_types, + legacy_4w4c_fp_layouts), + q_dq_8bit_reference_impl}; +} + +} // namespace q8ta_qdq +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/test_sets/unary_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/test_sets/unary_sets.cpp new file mode 100644 index 00000000000..2d7340ee916 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/test_sets/unary_sets.cpp @@ -0,0 +1,120 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_unary { + +namespace { + +std::vector generate_test_cases( + const std::vector>& shapes, + const std::vector& fp_layouts, + const std::vector& quant_layouts, + const std::vector& storage_types) { + std::vector test_cases; + + for (const auto& shape : shapes) { + std::string prefix = "ACCU"; + for (const auto& dim : shape) { + if (dim > kRefDimSizeLimit) { + prefix = "PERF"; + break; + } + } + + for (const auto& fp_layout : fp_layouts) { + for (const auto& quant_layout : quant_layouts) { + for (const auto& storage_type : storage_types) { + Q8taUnaryConfig config; + config.shape = shape; + config.test_case_name = prefix; + + test_cases.push_back(create_test_case_from_config( + config, storage_type, vkapi::kFloat, fp_layout, quant_layout)); + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("q8ta/unary", "correctness") { + const std::vector> shapes = { + {1, 3, 16, 16}, + {1, 8, 32, 32}, + {1, 16, 24, 24}, + {1, 32, 12, 12}, + {1, 1, 64, 64}, + {1, 3, 64, 64}, + {1, 4, 16, 16}, + + {1, 8, 20, 20}, + {1, 16, 14, 14}, + {1, 8, 28, 28}, + + // Odd tensor sizes + {1, 3, 15, 15}, + {1, 13, 31, 31}, + {1, 17, 23, 23}, + + // Larger tensors + {1, 64, 128, 128}, + {1, 32, 64, 64}, + {1, 128, 56, 56}, + {1, 128, 128, 128}, + }; + const std::vector fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + const std::vector storage_types = {utils::kBuffer}; + return { + generate_test_cases(shapes, fp_layouts, quant_layouts, storage_types), + q8ta_unary_reference_impl}; +} + +REGISTER_TEST_CASE_SET("q8ta/unary", "debug") { + const std::vector> shapes = { + {1, 16, 16, 16}, + }; + const std::vector fp_layouts = { + utils::kWidthPacked, + utils::kChannelsPacked, + }; + const std::vector quant_layouts = { + utils::kPackedInt8_4W, + utils::kPackedInt8_4C, + utils::kPackedInt8_4W4C, + utils::kPackedInt8_4H4W, + utils::kPackedInt8_4C1W, + }; + const std::vector storage_types = {utils::kBuffer}; + return { + generate_test_cases(shapes, fp_layouts, quant_layouts, storage_types), + q8ta_unary_reference_impl}; +} + +} // namespace q8ta_unary +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/test_q8ta_unary.cpp b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/unary.cpp similarity index 55% rename from backends/vulkan/test/custom_ops/test_q8ta_unary.cpp rename to backends/vulkan/test/ops/benchmark/cases/q8ta/unary/unary.cpp index a4072c01f02..d11d5c41a79 100644 --- a/backends/vulkan/test/custom_ops/test_q8ta_unary.cpp +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/unary.cpp @@ -4,28 +4,17 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -#include -#include +#include + #include #include -#include -#include -#include "utils.h" - -#include - -// #define DEBUG_MODE - -using namespace executorch::vulkan::prototyping; -using namespace vkcompute; - -static constexpr int64_t kRefDimSizeLimit = 512; +#include +#include -struct Q8taUnaryConfig { - std::vector shape; - std::string test_case_name = "placeholder"; - std::string op_name = "q8ta_unary_test"; -}; +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_unary { TestCase create_test_case_from_config( const Q8taUnaryConfig& config, @@ -101,113 +90,6 @@ TestCase create_test_case_from_config( return test_case; } -std::vector generate_q8ta_unary_easy_cases() { - std::vector test_cases; - - Q8taUnaryConfig config = { - {1, 16, 16, 16}, - "ACCU", - }; - - std::vector fp_layouts = { - utils::kWidthPacked, - utils::kChannelsPacked, - }; - - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - std::vector storage_types = {utils::kBuffer}; - std::vector float_types = {vkapi::kFloat}; - - for (const auto& fp_layout : fp_layouts) { - for (const auto& quant_layout : quant_layouts) { - for (const auto& storage_type : storage_types) { - for (const auto& input_dtype : float_types) { - test_cases.push_back(create_test_case_from_config( - config, storage_type, input_dtype, fp_layout, quant_layout)); - } - } - } - } - - return test_cases; -} - -std::vector generate_q8ta_unary_test_cases() { - std::vector test_cases; - - std::vector> shapes = { - {1, 3, 16, 16}, - {1, 8, 32, 32}, - {1, 16, 24, 24}, - {1, 32, 12, 12}, - {1, 1, 64, 64}, - {1, 3, 64, 64}, - {1, 4, 16, 16}, - - {1, 8, 20, 20}, - {1, 16, 14, 14}, - {1, 8, 28, 28}, - - // Odd tensor sizes - {1, 3, 15, 15}, - {1, 13, 31, 31}, - {1, 17, 23, 23}, - - // Larger tensors - {1, 64, 128, 128}, - {1, 32, 64, 64}, - {1, 128, 56, 56}, - {1, 128, 128, 128}, - }; - - std::vector fp_layouts = { - utils::kWidthPacked, - utils::kChannelsPacked, - }; - - std::vector quant_layouts = { - utils::kPackedInt8_4W, - utils::kPackedInt8_4C, - utils::kPackedInt8_4W4C, - utils::kPackedInt8_4H4W, - utils::kPackedInt8_4C1W, - }; - - std::vector storage_types = {utils::kBuffer}; - - for (const auto& shape : shapes) { - std::string prefix = "ACCU"; - for (const auto& dim : shape) { - if (dim > kRefDimSizeLimit) { - prefix = "PERF"; - break; - } - } - - for (const auto& fp_layout : fp_layouts) { - for (const auto& quant_layout : quant_layouts) { - for (const auto& storage_type : storage_types) { - Q8taUnaryConfig config; - config.shape = shape; - config.test_case_name = prefix; - - test_cases.push_back(create_test_case_from_config( - config, storage_type, vkapi::kFloat, fp_layout, quant_layout)); - } - } - } - } - - return test_cases; -} - // Reference implementation: quantize -> relu -> dequantize void q8ta_unary_reference_impl(TestCase& test_case) { int32_t idx = 0; @@ -280,32 +162,7 @@ void q8ta_unary_reference_impl(TestCase& test_case) { } } -int main(int argc, char* argv[]) { - set_debugging(false); - set_print_output(false); -#ifdef DEBUG_MODE - set_print_latencies(false); -#else - set_print_latencies(false); -#endif - set_use_gpu_timestamps(true); - - print_performance_header(); - std::cout << "Q8TA Unary (ReLU) Operation Prototyping Framework" << std::endl; - print_separator(); - - ReferenceComputeFunc ref_fn = q8ta_unary_reference_impl; - - auto results = execute_test_cases( -#ifdef DEBUG_MODE - generate_q8ta_unary_easy_cases, -#else - generate_q8ta_unary_test_cases, -#endif - "Q8taUnary", - /*warmup_runs = */ 1, - /*benchmark_runs = */ 1, - ref_fn); - - return 0; -} +} // namespace q8ta_unary +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/unary.h b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/unary.h new file mode 100644 index 00000000000..2ff17187a8e --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/q8ta/unary/unary.h @@ -0,0 +1,40 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace q8ta_unary { + +constexpr int64_t kRefDimSizeLimit = 512; + +struct Q8taUnaryConfig { + std::vector shape; + std::string test_case_name = "placeholder"; + std::string op_name = "q8ta_unary_test"; +}; + +TestCase create_test_case_from_config( + const Q8taUnaryConfig& config, + utils::StorageType storage_type, + vkapi::ScalarType input_dtype, + utils::GPUMemoryLayout fp_memory_layout, + utils::GPUMemoryLayout quant_layout); + +void q8ta_unary_reference_impl(TestCase& test_case); + +} // namespace q8ta_unary +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.cpp b/backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.cpp new file mode 100644 index 00000000000..24e39a3e110 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.cpp @@ -0,0 +1,202 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace sdpa { + +TestCase create_sdpa_test_case( + const SDPAConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + const std::string& impl) { + TestCase test_case; + + const bool is_perf = config.context_len > kRefContextLenLimit; + const std::string prefix = is_perf ? "PERF" : "ACCU"; + const std::string storage_str = repr_str(storage_type, utils::kWidthPacked); + const std::string dtype_str = dtype_short(dtype); + + const std::string shape = "D" + std::to_string(config.head_dim) + " H" + + std::to_string(config.n_heads) + " Hkv" + + std::to_string(config.n_kv_heads) + " S" + + std::to_string(config.seq_len) + " C" + + std::to_string(config.context_len); + + const std::string suffix = + "[" + config.model + " " + config.regime + " " + impl + "]"; + + test_case.set_name(make_test_label( + prefix, dtype_str, dtype_str, shape, storage_str, suffix)); + test_case.set_operator_name("test_etvk.test_sdpa.default"); + + // q: [1, S, n_heads, head_dim] + ValueSpec q( + {1, config.seq_len, config.n_heads, config.head_dim}, + dtype, + storage_type, + utils::kWidthPacked, + DataGenType::RANDOM); + + // k_cache / v_cache: [1, context_len, n_kv_heads, head_dim] + ValueSpec k_cache( + {1, config.context_len, config.n_kv_heads, config.head_dim}, + dtype, + storage_type, + utils::kWidthPacked, + DataGenType::RANDOM); + ValueSpec v_cache( + {1, config.context_len, config.n_kv_heads, config.head_dim}, + dtype, + storage_type, + utils::kWidthPacked, + DataGenType::RANDOM); + + ValueSpec impl_selector = ValueSpec::make_string(impl); + + // out: [1, S, n_heads, head_dim] + ValueSpec output( + {1, config.seq_len, config.n_heads, config.head_dim}, + dtype, + storage_type, + utils::kWidthPacked, + DataGenType::ZEROS); + + test_case.add_input_spec(q); + test_case.add_input_spec(k_cache); + test_case.add_input_spec(v_cache); + test_case.add_input_spec(impl_selector); + test_case.add_output_spec(output); + + if (dtype == vkapi::kHalf) { + test_case.set_abs_tolerance(1e-2f); + test_case.set_rel_tolerance(1e-2f); + } else { + test_case.set_abs_tolerance(1e-3f); + test_case.set_rel_tolerance(1e-3f); + } + + return test_case; +} + +// FLOPs: QK (2*S*C*D) + AV (2*S*C*D) per head, summed over heads. Softmax +// is negligible. Uses the causal-average context (~C/2) is ignored; report +// full-C dense FLOPs as an upper bound proxy. +int64_t sdpa_flop_calculator(const TestCase& test_case) { + const auto q_sizes = test_case.inputs()[0].get_tensor_sizes(); + const auto k_sizes = test_case.inputs()[1].get_tensor_sizes(); + const int64_t S = q_sizes[1]; + const int64_t H = q_sizes[2]; + const int64_t D = q_sizes[3]; + const int64_t C = k_sizes[1]; + return 4 * H * S * C * D; +} + +static std::vector as_float_data(const ValueSpec& spec) { + if (spec.dtype == vkapi::kFloat) { + return spec.get_float_data(); + } + if (spec.dtype == vkapi::kHalf) { + const auto& half_bits = spec.get_half_data(); + std::vector out(half_bits.size()); + for (size_t i = 0; i < half_bits.size(); ++i) { + out[i] = half_to_float(half_bits[i]); + } + return out; + } + throw std::invalid_argument("as_float_data: unsupported dtype"); +} + +// Reference: causal SDPA over the KV cache. +// q:[1,S,H,D], k/v cache:[1,C,Hkv,D], input_pos = C - S. +// For query row s (absolute position input_pos + s), attends to cache +// positions [0, input_pos + s]. GQA: head h maps to kv head h / (H/Hkv). +void sdpa_reference_impl(TestCase& test_case) { + const auto& q = test_case.inputs()[0]; + const auto& k = test_case.inputs()[1]; + const auto& v = test_case.inputs()[2]; + + const auto q_sizes = q.get_tensor_sizes(); + const auto k_sizes = k.get_tensor_sizes(); + + const int64_t S = q_sizes[1]; + const int64_t H = q_sizes[2]; + const int64_t D = q_sizes[3]; + const int64_t C = k_sizes[1]; + const int64_t Hkv = k_sizes[2]; + + if (C > kRefContextLenLimit) { + throw std::invalid_argument("sdpa reference: perf shape, skipping"); + } + + const int64_t input_pos = C - S; + const int64_t heads_per_kv = H / Hkv; + const float scale = 1.0f / std::sqrt(static_cast(D)); + + const auto q_data = as_float_data(q); + const auto k_data = as_float_data(k); + const auto v_data = as_float_data(v); + + ValueSpec& output = test_case.outputs()[0]; + auto& ref = output.get_ref_float_data(); + ref.assign(S * H * D, 0.0f); + + // Index helpers (contiguous WHCN-flattened as [1, dim1, dim2, dim3]). + auto q_idx = [&](int64_t s, int64_t h, int64_t d) { + return (s * H + h) * D + d; + }; + auto kv_idx = [&](int64_t c, int64_t hk, int64_t d) { + return (c * Hkv + hk) * D + d; + }; + + std::vector scores(C); + for (int64_t s = 0; s < S; ++s) { + const int64_t attend_len = input_pos + s + 1; // causal + for (int64_t h = 0; h < H; ++h) { + const int64_t hk = h / heads_per_kv; + + float max_score = -std::numeric_limits::infinity(); + for (int64_t c = 0; c < attend_len; ++c) { + float dot = 0.0f; + for (int64_t d = 0; d < D; ++d) { + dot += q_data[q_idx(s, h, d)] * k_data[kv_idx(c, hk, d)]; + } + dot *= scale; + scores[c] = dot; + max_score = std::max(max_score, dot); + } + + float denom = 0.0f; + for (int64_t c = 0; c < attend_len; ++c) { + scores[c] = std::exp(scores[c] - max_score); + denom += scores[c]; + } + + for (int64_t d = 0; d < D; ++d) { + float acc = 0.0f; + for (int64_t c = 0; c < attend_len; ++c) { + acc += scores[c] * v_data[kv_idx(c, hk, d)]; + } + ref[q_idx(s, h, d)] = acc / denom; + } + } + } +} + +} // namespace sdpa +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.h b/backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.h new file mode 100644 index 00000000000..a04e2784704 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/sdpa/sdpa.h @@ -0,0 +1,50 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace sdpa { + +// Correctness is only checked for these small shapes; larger perf shapes throw +// std::invalid_argument from the reference (framework marks them SKIPPED) to +// avoid an O(H*S*context*D) CPU reference on the large perf matrix. +constexpr int64_t kRefContextLenLimit = 256; + +// LLM SDPA (llama.custom_sdpa) shape: +// q: [1, S, n_heads, head_dim] (DHSB, width-packed) +// k/v cache:[1, context_len, n_kv_heads, head_dim] +struct SDPAConfig { + int64_t head_dim; + int64_t n_heads; + int64_t n_kv_heads; + int64_t seq_len; // S: query tokens (1 for decode, >1 for prefill) + int64_t context_len; // total KV length (kv_len) + std::string model; // label only + std::string regime; // "decode" / "prefill", label only +}; + +TestCase create_sdpa_test_case( + const SDPAConfig& config, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + const std::string& impl); + +void sdpa_reference_impl(TestCase& test_case); + +int64_t sdpa_flop_calculator(const TestCase& test_case); + +} // namespace sdpa +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/cases/sdpa/test_sets/sdpa_sets.cpp b/backends/vulkan/test/ops/benchmark/cases/sdpa/test_sets/sdpa_sets.cpp new file mode 100644 index 00000000000..95b1b341713 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/cases/sdpa/test_sets/sdpa_sets.cpp @@ -0,0 +1,180 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { +namespace sdpa { + +namespace { + +struct ModelDims { + std::string name; + int64_t head_dim; + int64_t n_heads; + int64_t n_kv_heads; +}; + +const std::vector kModels = { + {"Llama-3.2-1B", 64, 32, 8}, + {"Qwen3-0.6B", 128, 16, 8}, + {"Phi-4-mini", 128, 24, 8}, +}; + +// fp32 cases: each config yields one case per storage type x impl. +struct SDPACaseGroup { + std::vector configs; + std::vector storage_types; + std::vector impls; +}; + +struct SDPASetOptions { + // Swept over every model in kModels (fp16, texture). Each decode context + // length yields an S=1 case per decode impl; each prefill sequence length + // yields one S == context_len case. + std::vector decode_context_lens = {}; + std::vector decode_impls = {}; + std::vector prefill_seq_lens = {}; + // Appended after the model sweep. + std::vector accu_cases = {}; +}; + +std::vector generate_test_cases(const SDPASetOptions& options) { + std::vector test_cases; + + // Perf runs use fp16 texture (matches the LLM decode/prefill production + // path). A couple of small ACCU shapes validate correctness in fp32. + const auto dtype = vkapi::kHalf; + const auto storage = utils::kTexture3D; + + for (const auto& m : kModels) { + for (int64_t c : options.decode_context_lens) { + SDPAConfig cfg; + cfg.head_dim = m.head_dim; + cfg.n_heads = m.n_heads; + cfg.n_kv_heads = m.n_kv_heads; + cfg.seq_len = 1; + cfg.context_len = c; + cfg.model = m.name; + cfg.regime = "decode"; + for (const auto& impl : options.decode_impls) { + test_cases.push_back(create_sdpa_test_case(cfg, dtype, storage, impl)); + } + } + for (int64_t s : options.prefill_seq_lens) { + SDPAConfig cfg; + cfg.head_dim = m.head_dim; + cfg.n_heads = m.n_heads; + cfg.n_kv_heads = m.n_kv_heads; + cfg.seq_len = s; + cfg.context_len = s; + cfg.model = m.name; + cfg.regime = "prefill"; + // Prefill (tiled) is unaffected by the impl selector, so it runs a + // single case. + test_cases.push_back( + create_sdpa_test_case(cfg, dtype, storage, "default")); + } + } + + for (const auto& group : options.accu_cases) { + for (const auto& cfg : group.configs) { + for (const auto& st : group.storage_types) { + for (const auto& impl : group.impls) { + test_cases.push_back( + create_sdpa_test_case(cfg, vkapi::kFloat, st, impl)); + } + } + } + } + + return test_cases; +} + +} // namespace + +REGISTER_TEST_CASE_SET("sdpa", "correctness") { + return { + generate_test_cases({ + // Prefill: S == context_len. + .prefill_seq_lens = {128}, + // Small ACCU correctness cases (fp32), decode + prefill. Texture is + // the production LLM path; buffer is also validated for decode to + // guard the attn_weights S/context alignment (a decode-shaped buffer + // allocation has no headroom for the shaders' align_up_4 stride + // unless padded — see sdpa_impl). + .accu_cases = + { + // Cover D=64 (D4=16) and D=128 (D4=32) with the + // vendor-default GQA and the per-query-head shaders, across + // texture + buffer. + {.configs = + { + {64, 8, 2, 1, 32, "accu", "decode"}, + {128, 8, 2, 1, 32, "accu_d128", "decode"}, + }, + .storage_types = {utils::kTexture3D, utils::kBuffer}, + .impls = {"gqa", "non_gqa"}}, + // Force the head_dim output-tiled GQA variant (Adreno-only + // in production) so its wg x-collapse and the + // partial_n_tile tail get deterministic coverage on any + // device: D=64/128 give even D4 (fast path); D=4 gives D4=1 + // (odd), exercising the partial-tile checked load. + {.configs = + { + {64, 8, 2, 1, 32, "accu_tile2", "decode"}, + {128, 8, 2, 1, 32, "accu_tile2_d128", "decode"}, + {4, 8, 2, 1, 32, "accu_tile2_d4", "decode"}, + }, + .storage_types = {utils::kTexture3D, utils::kBuffer}, + .impls = {"gqa_tile2"}}, + {.configs = {{64, 8, 2, 16, 16, "accu", "prefill"}}, + .storage_types = {utils::kTexture3D}, + .impls = {"default"}}, + }, + }), + sdpa_reference_impl, + sdpa_flop_calculator, + /*warmup_runs=*/3, + /*benchmark_runs=*/10}; +} + +REGISTER_TEST_CASE_SET("sdpa", "decode") { + return { + generate_test_cases({ + // Decode: S=1, sweep context_len. + .decode_context_lens = {512, 1024, 4096}, + // Decode (S==1) picks a coop AV shader; exercise both the GQA-reuse + // variant and the per-query-head variant for every decode case. + .decode_impls = {"gqa", "non_gqa"}, + }), + sdpa_reference_impl, + sdpa_flop_calculator, + /*warmup_runs=*/10, + /*benchmark_runs=*/30}; +} + +REGISTER_TEST_CASE_SET("sdpa", "performance") { + return { + generate_test_cases({ + // Prefill: S == context_len. + .prefill_seq_lens = {512}, + }), + sdpa_reference_impl, + sdpa_flop_calculator, + /*warmup_runs=*/3, + /*benchmark_runs=*/10}; +} + +} // namespace sdpa +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/etvk_bench.cpp b/backends/vulkan/test/ops/benchmark/etvk_bench.cpp new file mode 100644 index 00000000000..9b642d8dc2d --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/etvk_bench.cpp @@ -0,0 +1,247 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +// Runs the test case sets registered under cases/. See --helpshort for usage. + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +DEFINE_bool(list, false, "List the selected sets as : and exit."); +DEFINE_string( + op, + "", + "Comma-separated operators whose sets to run. An operator is its " + "directory under cases/; a parent directory such as q8ta selects every " + "operator beneath it. Empty selects all operators."); +DEFINE_string( + set, + "", + "Comma-separated set names to run. Empty selects all sets."); +DEFINE_string( + filter, + "", + "Run only test cases whose name matches the filter: '|'-separated " + "alternatives, each a ';'-separated list of substrings that must all " + "appear. For example, 'f16;+bias|f32' selects f16 cases with bias and all " + "f32 cases."); +DEFINE_int32( + chain, + 0, + "Dispatch the operator this many times per measured execute. 0 sizes the " + "chain adaptively."); +DEFINE_bool( + gpu_timestamps, + true, + "Time with GPU timestamps. --nogpu_timestamps uses the wall clock."); +DEFINE_bool(print_output, false, "Print output tensor data."); +DEFINE_bool(print_latencies, false, "Print per-iteration latencies."); +DEFINE_bool(debug, false, "Print debugging information."); +DEFINE_bool(device_info, false, "Print the Vulkan device properties and exit."); + +using namespace executorch::vulkan::prototyping; + +namespace { + +std::vector split(const std::string& value, char delimiter) { + std::vector items; + std::stringstream stream(value); + std::string item; + while (std::getline(stream, item, delimiter)) { + if (!item.empty()) { + items.push_back(item); + } + } + return items; +} + +bool op_selected(const std::vector& ops, const std::string& op) { + if (ops.empty()) { + return true; + } + for (const std::string& sel : ops) { + if (op == sel || op.rfind(sel + "/", 0) == 0) { + return true; + } + } + return false; +} + +bool set_selected( + const std::vector& sets, + const std::string& name) { + return sets.empty() || + std::find(sets.begin(), sets.end(), name) != sets.end(); +} + +// One list of required substrings per alternative. +using NameFilter = std::vector>; + +NameFilter parse_filter(const std::string& value) { + NameFilter filter; + for (const std::string& alternative : split(value, '|')) { + std::vector substrings = split(alternative, ';'); + if (!substrings.empty()) { + filter.push_back(std::move(substrings)); + } + } + return filter; +} + +bool contains_all( + const std::string& name, + const std::vector& substrings) { + for (const std::string& substring : substrings) { + if (name.find(substring) == std::string::npos) { + return false; + } + } + return true; +} + +bool name_matches(const NameFilter& filter, const std::string& name) { + if (filter.empty()) { + return true; + } + for (const std::vector& substrings : filter) { + if (contains_all(name, substrings)) { + return true; + } + } + return false; +} + +bool set_precedes( + const RegisteredTestCaseSet* a, + const RegisteredTestCaseSet* b) { + return a->op != b->op ? a->op < b->op : a->name < b->name; +} + +std::vector sorted_sets() { + std::vector sets; + for (const RegisteredTestCaseSet& set : registered_test_case_sets()) { + sets.push_back(&set); + } + std::sort(sets.begin(), sets.end(), set_precedes); + return sets; +} + +// Returns true if the set passed. +bool run_set( + const RegisteredTestCaseSet& registered, + const NameFilter& filter) { + const std::string id = registered.op + ":" + registered.name; + print_performance_header(); + std::cout << id << std::endl; + print_separator(); + + try { + TestCaseSet set = registered.make(); + if (set.custom_test) { + set.custom_test(); + return true; + } + std::vector test_cases; + for (TestCase& test_case : set.test_cases) { + if (!name_matches(filter, test_case.name())) { + continue; + } + if (FLAGS_chain > 0) { + test_case.set_op_invocations_per_execute(FLAGS_chain); + } + test_cases.push_back(std::move(test_case)); + } + execute_test_cases( + std::move(test_cases), + set.flop_calculator, + id, + set.warmup_runs, + set.benchmark_runs, + set.reference); + } catch (const std::exception& e) { + std::cerr << id << " failed: " << e.what() << std::endl; + return false; + } + return true; +} + +} // namespace + +int main(int argc, char* argv[]) { + gflags::SetUsageMessage( + "Runs the test case sets registered under cases/. With no selection " + "flags, every set runs."); + gflags::ParseCommandLineFlags(&argc, &argv, true); + if (argc > 1) { + std::cerr << "Unexpected argument: " << argv[1] << std::endl; + return 2; + } + if (FLAGS_chain < 0) { + std::cerr << "--chain must not be negative" << std::endl; + return 2; + } + + if (FLAGS_device_info) { + print_device_info(); + return 0; + } + + const std::vector ops = split(FLAGS_op, ','); + const std::vector set_names = split(FLAGS_set, ','); + std::vector selected; + for (const RegisteredTestCaseSet* set : sorted_sets()) { + if (op_selected(ops, set->op) && set_selected(set_names, set->name)) { + selected.push_back(set); + } + } + if (selected.empty()) { + std::cerr << "No test case sets match the selection" << std::endl; + return 2; + } + + if (FLAGS_list) { + for (const RegisteredTestCaseSet* set : selected) { + std::cout << set->op << ":" << set->name << std::endl; + } + return 0; + } + + set_debugging(FLAGS_debug); + set_print_output(FLAGS_print_output); + set_print_latencies(FLAGS_print_latencies); + set_use_gpu_timestamps(FLAGS_gpu_timestamps); + + const NameFilter filter = parse_filter(FLAGS_filter); + std::vector failed; + for (const RegisteredTestCaseSet* set : selected) { + if (!run_set(*set, filter)) { + failed.push_back(set->op + ":" + set->name); + } + } + + print_separator(); + std::cout << "Ran " << selected.size() << " test case sets"; + if (failed.empty()) { + std::cout << ", all passed" << std::endl; + } else { + std::cout << ", " << failed.size() << " failed" << std::endl; + } + for (const std::string& id : failed) { + std::cout << " FAILED: " << id << std::endl; + } + return failed.empty() ? 0 : 1; +} diff --git a/backends/vulkan/test/custom_ops/cm_utils.cpp b/backends/vulkan/test/ops/benchmark/framework/cm_utils.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/cm_utils.cpp rename to backends/vulkan/test/ops/benchmark/framework/cm_utils.cpp diff --git a/backends/vulkan/test/custom_ops/cm_utils.h b/backends/vulkan/test/ops/benchmark/framework/cm_utils.h similarity index 100% rename from backends/vulkan/test/custom_ops/cm_utils.h rename to backends/vulkan/test/ops/benchmark/framework/cm_utils.h diff --git a/backends/vulkan/test/ops/benchmark/framework/config.cpp b/backends/vulkan/test/ops/benchmark/framework/config.cpp new file mode 100644 index 00000000000..ada88cb840b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/config.cpp @@ -0,0 +1,55 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "config.h" + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// Output and latency printing utilities +namespace { +static int print_output_enabled = 0; +static int print_latencies_enabled = 0; +static int gpu_timestamps_enabled = 0; +static int debugging_enabled = 0; +} // namespace + +bool print_output() { + return print_output_enabled > 0; +} + +void set_print_output(bool print_output) { + print_output_enabled = print_output ? 1 : 0; +} + +bool print_latencies() { + return print_latencies_enabled > 0; +} + +void set_print_latencies(bool print_latencies) { + print_latencies_enabled = print_latencies ? 1 : 0; +} + +bool use_gpu_timestamps() { + return gpu_timestamps_enabled > 0; +} + +void set_use_gpu_timestamps(bool use_timestamps) { + gpu_timestamps_enabled = use_timestamps ? 1 : 0; +} + +bool debugging() { + return debugging_enabled > 0; +} + +void set_debugging(bool enable_debugging) { + debugging_enabled = enable_debugging ? 1 : 0; +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/config.h b/backends/vulkan/test/ops/benchmark/framework/config.h new file mode 100644 index 00000000000..6b2a3475234 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/config.h @@ -0,0 +1,31 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// +// Global configuration options +// + +bool print_output(); +void set_print_output(bool print_output); + +bool print_latencies(); +void set_print_latencies(bool print_latencies); + +bool use_gpu_timestamps(); +void set_use_gpu_timestamps(bool use_timestamps); + +bool debugging(); +void set_debugging(bool enable_debugging); + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/custom_ops/conv2d_utils.cpp b/backends/vulkan/test/ops/benchmark/framework/conv2d_utils.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/conv2d_utils.cpp rename to backends/vulkan/test/ops/benchmark/framework/conv2d_utils.cpp diff --git a/backends/vulkan/test/custom_ops/conv2d_utils.h b/backends/vulkan/test/ops/benchmark/framework/conv2d_utils.h similarity index 100% rename from backends/vulkan/test/custom_ops/conv2d_utils.h rename to backends/vulkan/test/ops/benchmark/framework/conv2d_utils.h diff --git a/backends/vulkan/test/custom_ops/test_coopmat_probe.cpp b/backends/vulkan/test/ops/benchmark/framework/device_info.cpp similarity index 77% rename from backends/vulkan/test/custom_ops/test_coopmat_probe.cpp rename to backends/vulkan/test/ops/benchmark/framework/device_info.cpp index 257a5df95fc..5275b7c6121 100644 --- a/backends/vulkan/test/custom_ops/test_coopmat_probe.cpp +++ b/backends/vulkan/test/ops/benchmark/framework/device_info.cpp @@ -4,20 +4,18 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -// One-shot capability probe: prints the cooperative matrix configurations -// exposed by the active Vulkan device, plus the relevant adapter properties -// (subgroup size, iGPU vs dGPU, device name). Used to gate the coopmat -// implementations for quantized linear shaders. +#include "device_info.h" +#include "cm_utils.h" #include -#include "cm_utils.h" - #include -using namespace executorch::vulkan::prototyping; +namespace executorch { +namespace vulkan { +namespace prototyping { -int main() { +void print_device_info() { auto* adapter = vkcompute::api::context()->adapter_ptr(); std::cout << "=== Vulkan adapter ===\n"; @@ -36,6 +34,8 @@ int main() { << (adapter->supports_int8_dot_product() ? "yes" : "no") << "\n"; queryCooperativeMatrixProperties(); - - return 0; } + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/device_info.h b/backends/vulkan/test/ops/benchmark/framework/device_info.h new file mode 100644 index 00000000000..9094ca8d5cb --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/device_info.h @@ -0,0 +1,19 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// Prints the active Vulkan adapter's properties and the cooperative matrix +// configurations it exposes. +void print_device_info(); + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/labels.h b/backends/vulkan/test/ops/benchmark/framework/labels.h new file mode 100644 index 00000000000..bcae57ccef8 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/labels.h @@ -0,0 +1,136 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// NOLINTNEXTLINE(google-build-using-namespace) +using namespace vkcompute; + +// +// String utilities +// + +// Helper function to get abbreviated layout names for test case naming +inline std::string layout_abbrev(utils::GPUMemoryLayout layout) { + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (layout) { + case utils::kWidthPacked: + return "WP"; + case utils::kChannelsPacked: + return "CP"; + case utils::kPackedInt8_4W: + return "4W"; + case utils::kPackedInt8_4C: + return "4C"; + case utils::kPackedInt8_4W4C: + return "4W4C"; + case utils::kPackedInt8_4H4W: + return "4H4W"; + case utils::kPackedInt8_4C1W: + return "4C1W"; + default: + return "UNK"; + } +} + +// Helper function to get abbreviated storage type names for test case naming +inline std::string storage_type_abbrev(utils::StorageType storage_type) { + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (storage_type) { + case utils::kTexture3D: + return "Tex"; + case utils::kBuffer: + return "Buf"; + default: + return "UNK"; + } +} + +// Helper function to get combined storage type and layout representation +// Example: (kBuffer, kPackedInt8_4W4C) -> "Buf_4W4C" +inline std::string repr_str( + utils::StorageType storage_type, + utils::GPUMemoryLayout layout) { + return storage_type_abbrev(storage_type) + "(" + layout_abbrev(layout) + ")"; +} + +// Helper function to generate comma-separated shape string for test case naming +// Example: {1, 128, 56, 56} -> "1,128,56,56" +inline std::string shape_string(const std::vector& shape) { + std::string result; + for (size_t i = 0; i < shape.size(); ++i) { + if (i > 0) { + result += ","; + } + result += std::to_string(shape[i]); + } + return result; +} + +// Helper function to generate a bracketed shape string for test case naming +// Example: {1, 128, 56, 56} -> "[1,128,56,56]" +inline std::string shape_bracket(const std::vector& shape) { + return "[" + shape_string(shape) + "]"; +} + +// Short dtype symbol used in standardized test case labels +// Example: kFloat -> "f32", kHalf -> "f16", kChar -> "i8", kInt -> "i32" +inline std::string dtype_short(vkapi::ScalarType dtype) { + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + return "f32"; + case vkapi::kHalf: + return "f16"; + case vkapi::kChar: + return "i8"; + case vkapi::kByte: + return "u8"; + case vkapi::kInt: + return "i32"; + case vkapi::kBool: + return "b"; + default: + return "?"; + } +} + +// Build a standardized test label of the form: +// " [ ]" +// where is "->" when the two differ, or just +// "" when they match. Sections are separated by two spaces. If +// suffix is non-empty it is appended after a single space (allowing callers +// to pass e.g. "[general]" or "[gemv] +bias"). +inline std::string make_test_label( + const std::string& prefix, + const std::string& in_dtype, + const std::string& out_dtype, + const std::string& shape_str, + const std::string& storage_str, + const std::string& suffix = "") { + const std::string dtype_str = + (in_dtype == out_dtype) ? in_dtype : in_dtype + "->" + out_dtype; + std::string label = + prefix + " " + dtype_str + " " + shape_str + " " + storage_str; + if (!suffix.empty()) { + label += " " + suffix; + } + return label; +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/registry.cpp b/backends/vulkan/test/ops/benchmark/framework/registry.cpp new file mode 100644 index 00000000000..da37f972e70 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/registry.cpp @@ -0,0 +1,48 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "registry.h" + +#include + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +namespace { + +std::vector& registry() { + static std::vector sets; + return sets; +} + +} // namespace + +const std::vector& registered_test_case_sets() { + return registry(); +} + +TestCaseSetRegistrar::TestCaseSetRegistrar( + std::string op, + std::string name, + std::function make) { + for (const RegisteredTestCaseSet& set : registry()) { + VK_CHECK_COND( + set.op != op || set.name != name, + "Test case set ", + op, + ":", + name, + " is registered more than once"); + } + registry().push_back({std::move(op), std::move(name), std::move(make)}); +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/registry.h b/backends/vulkan/test/ops/benchmark/framework/registry.h new file mode 100644 index 00000000000..ca24db5f47c --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/registry.h @@ -0,0 +1,67 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include "runner.h" +#include "test_case.h" + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +struct TestCaseSet { + std::vector test_cases; + ReferenceComputeFunc reference = nullptr; + FlopCalculatorFunc flop_calculator = default_flop_calculator; + int warmup_runs = 1; + int benchmark_runs = 1; + // Runs instead of test_cases for checks that are not expressible as test + // cases. Throws on failure. + std::function custom_test = nullptr; +}; + +struct RegisteredTestCaseSet { + // Path of the operator's directory under cases/, e.g. "q8ta/conv2d". + std::string op; + std::string name; + // Invoked only when the set is selected, since building test cases may + // query the device. + std::function make; +}; + +const std::vector& registered_test_case_sets(); + +class TestCaseSetRegistrar final { + public: + TestCaseSetRegistrar( + std::string op, + std::string name, + std::function make); +}; + +#define ETVK_TEST_CASE_SET_CONCAT_IMPL(a, b) a##b +#define ETVK_TEST_CASE_SET_CONCAT(a, b) ETVK_TEST_CASE_SET_CONCAT_IMPL(a, b) + +// Registers the TestCaseSet returned by the function body that follows. +#define REGISTER_TEST_CASE_SET(op, name) \ + static ::executorch::vulkan::prototyping::TestCaseSet \ + ETVK_TEST_CASE_SET_CONCAT(make_test_case_set_, __LINE__)(); \ + static const ::executorch::vulkan::prototyping::TestCaseSetRegistrar \ + ETVK_TEST_CASE_SET_CONCAT(test_case_set_registrar_, __LINE__)( \ + op, \ + name, \ + &ETVK_TEST_CASE_SET_CONCAT(make_test_case_set_, __LINE__)); \ + static ::executorch::vulkan::prototyping::TestCaseSet \ + ETVK_TEST_CASE_SET_CONCAT(make_test_case_set_, __LINE__)() + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/results.cpp b/backends/vulkan/test/ops/benchmark/framework/results.cpp new file mode 100644 index 00000000000..3a71c50237b --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/results.cpp @@ -0,0 +1,468 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "results.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// BenchmarkResult implementation +void BenchmarkResult::add_iter_timing(float time_us) { + iter_timings.push_back(time_us); +} + +void BenchmarkResult::add_shader_timing( + const std::string& shader_name, + float time_us, + const uint32_t gwg[3], + const uint32_t lwg[3]) { + // Find existing shader timing or create new one + for (auto& st : shader_timings_) { + if (st.shader_name == shader_name) { + st.iter_timings_us.push_back(time_us); + // Work group sizes should be consistent across iterations + return; + } + } + // Not found, create new entry + ShaderTiming new_timing; + new_timing.shader_name = shader_name; + new_timing.iter_timings_us.push_back(time_us); + new_timing.gwg[0] = gwg[0]; + new_timing.gwg[1] = gwg[1]; + new_timing.gwg[2] = gwg[2]; + new_timing.lwg[0] = lwg[0]; + new_timing.lwg[1] = lwg[1]; + new_timing.lwg[2] = lwg[2]; + shader_timings_.push_back(std::move(new_timing)); +} + +float BenchmarkResult::get_avg_time_us() const { + if (iter_timings.empty()) { + return 0.0f; + } + + float sum = 0.0f; + for (float timing : iter_timings) { + sum += timing; + } + return sum / iter_timings.size(); +} + +float BenchmarkResult::get_min_time_us() const { + if (iter_timings.empty()) { + return 0.0f; + } + + return *std::min_element(iter_timings.begin(), iter_timings.end()); +} + +float BenchmarkResult::get_max_time_us() const { + if (iter_timings.empty()) { + return 0.0f; + } + + return *std::max_element(iter_timings.begin(), iter_timings.end()); +} + +float BenchmarkResult::get_std_dev_us() const { + if (iter_timings.size() <= 1) { + return 0.0f; + } + + float mean = get_avg_time_us(); + float sum_sq_diff = 0.0f; + + for (float timing : iter_timings) { + float diff = timing - mean; + sum_sq_diff += diff * diff; + } + + return std::sqrt(sum_sq_diff / (iter_timings.size() - 1)); +} + +void BenchmarkResult::print_summary( + int case_number, + const std::string& size_info, + float total_gflops) const { + (void)case_number; + static constexpr int OPERATOR_NAME_WIDTH = 50; + static constexpr int GLOBAL_WG_WIDTH = 16; + static constexpr int LOCAL_WG_WIDTH = 12; + static constexpr int KERNEL_NAME_WIDTH = 80; + static constexpr int SIZE_INFO_WIDTH = 20; + static constexpr int TIMING_WIDTH = 16; + static constexpr int GFLOPS_WIDTH = 14; + static constexpr int CORRECTNESS_WIDTH = 8; + + // Helper to truncate shader names longer than 46 chars to 44 chars + ".." + auto truncate_shader_name = [](const std::string& name) -> std::string { + if (name.length() > 46) { + return name.substr(0, 44) + ".."; + } + return name; + }; + + // Helper to format work group size as (x,y,z) + auto format_wg_size = [](const uint32_t wg[3]) -> std::string { + return "(" + std::to_string(wg[0]) + "," + std::to_string(wg[1]) + "," + + std::to_string(wg[2]) + ")"; + }; + + std::string correctness_str; + switch (correctness_status_) { + case CorrectnessStatus::SKIPPED: + correctness_str = "SKIPPED"; + break; + case CorrectnessStatus::PASSED: + correctness_str = "PASSED"; + break; + case CorrectnessStatus::FAILED: + correctness_str = "FAILED"; + break; + } + + // If we have per-shader timing data, print one line per shader plus overall + if (!shader_timings_.empty()) { + // If only one shader, print a single combined row + if (shader_timings_.size() == 1) { + const auto& st = shader_timings_[0]; + std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) + << truncate_shader_name(st.shader_name) << " " << std::left + << std::setw(GLOBAL_WG_WIDTH) << format_wg_size(st.gwg) + << std::left << std::setw(LOCAL_WG_WIDTH) + << format_wg_size(st.lwg) << std::left + << std::setw(KERNEL_NAME_WIDTH) << get_kernel_name() + << std::right << " " << std::setw(SIZE_INFO_WIDTH) << size_info + << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) + << get_avg_time_us() << " μs " << std::setw(GFLOPS_WIDTH) + << std::fixed << std::setprecision(3) << total_gflops + << " GFLOP/s " << std::setw(CORRECTNESS_WIDTH) + << correctness_str << std::endl; + } else { + // Multiple shaders: print individual shader lines (without GFLOP/s) + for (size_t i = 0; i < shader_timings_.size(); ++i) { + const auto& st = shader_timings_[i]; + float shader_avg_time = st.get_avg_time_us(); + + // Shader lines don't show test case info + std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) + << truncate_shader_name(st.shader_name) << " " << std::left + << std::setw(GLOBAL_WG_WIDTH) << format_wg_size(st.gwg) + << std::left << std::setw(LOCAL_WG_WIDTH) + << format_wg_size(st.lwg) << std::left + << std::setw(KERNEL_NAME_WIDTH) << "" << std::right << " " + << std::setw(SIZE_INFO_WIDTH) << "" << std::setw(TIMING_WIDTH) + << std::fixed << std::setprecision(3) << shader_avg_time + << " μs " << std::setw(GFLOPS_WIDTH) << "" << " " + << std::setw(CORRECTNESS_WIDTH) << "" << std::endl; + } + + // Print overall row with operator name, test case info, total time, and + // GFLOP/s + std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) + << get_operator_name() << " " << std::left + << std::setw(GLOBAL_WG_WIDTH) << "" << std::left + << std::setw(LOCAL_WG_WIDTH) << "" << std::left + << std::setw(KERNEL_NAME_WIDTH) << get_kernel_name() + << std::right << " " << std::setw(SIZE_INFO_WIDTH) << size_info + << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) + << get_avg_time_us() << " μs " << std::setw(GFLOPS_WIDTH) + << std::fixed << std::setprecision(3) << total_gflops + << " GFLOP/s " << std::setw(CORRECTNESS_WIDTH) + << correctness_str << std::endl; + } + + // Print separator line between test cases + } else { + // No per-shader timing data, use the original format + std::cout << std::left << std::setw(OPERATOR_NAME_WIDTH) + << get_operator_name() << " " << std::left + << std::setw(GLOBAL_WG_WIDTH) << "" << std::left + << std::setw(LOCAL_WG_WIDTH) << "" << std::left + << std::setw(KERNEL_NAME_WIDTH) << get_kernel_name() << std::right + << " " << std::setw(SIZE_INFO_WIDTH) << size_info + << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) + << get_avg_time_us() << " μs " << std::setw(GFLOPS_WIDTH) + << std::fixed << std::setprecision(3) << total_gflops + << " GFLOP/s " << std::setw(CORRECTNESS_WIDTH) << correctness_str + << std::endl; + } +} + +// TestResult implementation +void TestResult::add_result(const BenchmarkResult& result) { + results_.push_back(result); +} + +void TestResult::add_result(BenchmarkResult&& result) { + results_.push_back(std::move(result)); +} + +void TestResult::print_summary() const { + static constexpr int CASE_WIDTH = 100; + static constexpr int KERNEL_NAME_WIDTH = 20; + static constexpr int TIMING_WIDTH = 12; + static constexpr int PASS_WIDTH = 8; + + if (results_.empty()) { + std::cout << "No results to display" << std::endl; + return; + } + + std::cout << "\n=== " << operation_name_ + << " Performance Summary ===" << std::endl; + print_separator(); + + std::cout << std::left << std::setw(CASE_WIDTH) << "Case" << std::left + << std::setw(KERNEL_NAME_WIDTH) << "Kernel Name" << std::left + << std::setw(TIMING_WIDTH) << "Avg (μs)" << std::left + << std::setw(TIMING_WIDTH) << "Min (μs)" << std::left + << std::setw(TIMING_WIDTH) << "Max (μs)" << std::left + << std::setw(TIMING_WIDTH) << "Std Dev" << std::left + << std::setw(PASS_WIDTH) << "Pass" << std::endl; + print_separator(); + + for (size_t i = 0; i < results_.size(); ++i) { + const auto& result = results_[i]; + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + std::cout << std::left << std::setw(CASE_WIDTH) << i + 1 << std::left + << std::setw(KERNEL_NAME_WIDTH) + << result.get_kernel_name().substr(0, KERNEL_NAME_WIDTH - 1) + << std::left << std::setw(TIMING_WIDTH) << std::fixed + << std::setprecision(3) << result.get_avg_time_us() << std::left + << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) + << result.get_min_time_us() << std::left + << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) + << result.get_max_time_us() << std::left + << std::setw(TIMING_WIDTH) << std::fixed << std::setprecision(3) + << result.get_std_dev_us() << std::left << std::setw(PASS_WIDTH) + << (vulkan_execute_succeeded ? "✓" : "✗") << std::endl; + } + + print_separator(); + std::cout << "Total cases: " << results_.size() + << ", Passed: " << get_passed_count() + << ", Failed: " << get_failed_count() << std::endl; + std::cout << "Overall GFLOP/s: " << std::fixed << std::setprecision(3) + << gflops_ << std::endl; + std::cout << "Overall correctness: " + << (correctness_passed_ ? "PASSED" : "FAILED") << std::endl; +} + +void TestResult::print_detailed_results() const { + if (results_.empty()) { + std::cout << "No results to display" << std::endl; + return; + } + + std::cout << "\n=== " << operation_name_ + << " Detailed Results ===" << std::endl; + + for (size_t i = 0; i < results_.size(); ++i) { + const auto& result = results_[i]; + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + std::cout << "\nCase " << i + 1 << ": " << result.get_kernel_name() + << std::endl; + std::cout << " Iterations: " << result.get_num_iterations() << std::endl; + std::cout << " Average: " << std::fixed << std::setprecision(3) + << result.get_avg_time_us() << " μs" << std::endl; + std::cout << " Min: " << std::fixed << std::setprecision(3) + << result.get_min_time_us() << " μs" << std::endl; + std::cout << " Max: " << std::fixed << std::setprecision(3) + << result.get_max_time_us() << " μs" << std::endl; + std::cout << " Std Dev: " << std::fixed << std::setprecision(3) + << result.get_std_dev_us() << " μs" << std::endl; + std::cout << " Correctness: " + << (vulkan_execute_succeeded ? "PASSED" : "FAILED") << std::endl; + + if (result.get_num_iterations() > 0) { + std::cout << " Individual timings (μs): "; + const auto& timings = result.get_iter_timings(); + for (size_t j = 0; j < std::min(size_t(10), timings.size()); ++j) { + std::cout << std::fixed << std::setprecision(1) << timings[j]; + if (j < std::min(size_t(10), timings.size()) - 1) { + std::cout << ", "; + } + } + if (timings.size() > 10) { + std::cout << " ... (" << (timings.size() - 10) << " more)"; + } + std::cout << std::endl; + } + } + + std::cout << "\nOverall Results:" << std::endl; + std::cout << " Total GFLOP/s: " << std::fixed << std::setprecision(3) + << gflops_ << std::endl; + std::cout << " Overall correctness: " + << (correctness_passed_ ? "PASSED" : "FAILED") << std::endl; +} + +void TestResult::print_statistics() const { + if (results_.empty()) { + std::cout << "No results to display statistics for" << std::endl; + return; + } + + std::cout << "\n=== " << operation_name_ << " Statistics ===" << std::endl; + std::cout << "Total test cases: " << results_.size() << std::endl; + std::cout << "Passed: " << get_passed_count() << std::endl; + std::cout << "Failed: " << get_failed_count() << std::endl; + std::cout << "Success rate: " << std::fixed << std::setprecision(1) + << (100.0f * get_passed_count() / results_.size()) << "%" + << std::endl; + + if (get_passed_count() > 0) { + std::cout << "Total average time: " << std::fixed << std::setprecision(3) + << get_total_avg_time_us() << " μs" << std::endl; + std::cout << "Total GFLOP/s: " << std::fixed << std::setprecision(3) + << get_total_gflops() << std::endl; + + const auto* fastest = get_fastest_result(); + const auto* slowest = get_slowest_result(); + const auto* highest_gflops = get_highest_gflops_result(); + + if (fastest) { + std::cout << "Fastest case: " << fastest->get_kernel_name() << " (" + << std::fixed << std::setprecision(3) + << fastest->get_avg_time_us() << " μs)" << std::endl; + } + + if (slowest) { + std::cout << "Slowest case: " << slowest->get_kernel_name() << " (" + << std::fixed << std::setprecision(3) + << slowest->get_avg_time_us() << " μs)" << std::endl; + } + + if (highest_gflops) { + std::cout << "Best performing case: " << highest_gflops->get_kernel_name() + << " (" << std::fixed << std::setprecision(3) + << highest_gflops->get_avg_time_us() << " μs)" << std::endl; + } + } +} + +void TestResult::print_brief_summary() const { + print_separator(); + std::cout << "Summary Statistics:" << std::endl; + + if (get_passed_count() > 0) { + std::cout << "Average execution time: " << std::fixed + << std::setprecision(3) << get_total_avg_time_us() << " μs" + << std::endl; + std::cout << "Total throughput: " << std::fixed << std::setprecision(3) + << get_gflops() << " GFLOP/s" << std::endl; + std::cout << "Successful test cases: " << get_passed_count() << "/" + << size() << std::endl; + std::cout << "Overall correctness: " + << (get_correctness_passed() ? "PASSED" : "FAILED") << std::endl; + } else { + std::cout << "No successful test cases to report" << std::endl; + } +} + +float TestResult::get_total_avg_time_us() const { + if (results_.empty()) { + return 0.0f; + } + + float sum = 0.0f; + size_t count = 0; + + for (const auto& result : results_) { + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + if (vulkan_execute_succeeded) { + sum += result.get_avg_time_us(); + count++; + } + } + + return count > 0 ? sum / count : 0.0f; +} + +float TestResult::get_total_gflops() const { + return gflops_; +} + +size_t TestResult::get_passed_count() const { + size_t count = 0; + for (const auto& result : results_) { + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + if (vulkan_execute_succeeded) { + count++; + } + } + return count; +} + +size_t TestResult::get_failed_count() const { + return results_.size() - get_passed_count(); +} + +const BenchmarkResult* TestResult::get_fastest_result() const { + const BenchmarkResult* fastest = nullptr; + + for (const auto& result : results_) { + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + if (vulkan_execute_succeeded) { + if (!fastest || result.get_avg_time_us() < fastest->get_avg_time_us()) { + fastest = &result; + } + } + } + + return fastest; +} + +const BenchmarkResult* TestResult::get_slowest_result() const { + const BenchmarkResult* slowest = nullptr; + + for (const auto& result : results_) { + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + if (vulkan_execute_succeeded) { + if (!slowest || result.get_avg_time_us() > slowest->get_avg_time_us()) { + slowest = &result; + } + } + } + + return slowest; +} + +const BenchmarkResult* TestResult::get_highest_gflops_result() const { + // Since GFLOPS is now a TestResult-level metric rather than per-case, + // this method returns the fastest result as a proxy for highest performance + return get_fastest_result(); +} + +// Utility functions for printing +void print_performance_header() { + std::cout << "\n=== Compute Shader Performance Benchmark ===" << std::endl; +} + +void print_separator() { + std::cout << std::string(70, '-') << std::endl; +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/results.h b/backends/vulkan/test/ops/benchmark/framework/results.h new file mode 100644 index 00000000000..e957ddf8bce --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/results.h @@ -0,0 +1,235 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// +// BenchmarkResult +// + +enum class CorrectnessStatus { + SKIPPED, // No reference function provided + PASSED, // Reference function provided and validation passed + FAILED // Reference function provided but validation failed +}; + +// Per-shader timing data for detailed reporting +struct ShaderTiming { + std::string shader_name; + std::vector iter_timings_us; // Individual iteration timings + uint32_t gwg[3] = {0, 0, 0}; + uint32_t lwg[3] = {0, 0, 0}; + + float get_avg_time_us() const { + if (iter_timings_us.empty()) { + return 0.0f; + } + float sum = 0.0f; + for (float t : iter_timings_us) { + sum += t; + } + return sum / iter_timings_us.size(); + } +}; + +class BenchmarkResult { + public: + BenchmarkResult() : correctness_status_(CorrectnessStatus::SKIPPED) {} + + explicit BenchmarkResult(const std::string& name) + : kernel_name(name), correctness_status_(CorrectnessStatus::SKIPPED) {} + + BenchmarkResult( + const std::string& kernel_name, + const std::string& operator_name) + : kernel_name(kernel_name), + operator_name(operator_name), + correctness_status_(CorrectnessStatus::SKIPPED) {} + + // Add timing for a single iteration + void add_iter_timing(float time_us); + + // Add per-shader timing for a single iteration + void add_shader_timing( + const std::string& shader_name, + float time_us, + const uint32_t gwg[3], + const uint32_t lwg[3]); + + // Get per-shader timing data + const std::vector& get_shader_timings() const { + return shader_timings_; + } + + // Getters + const std::string& get_kernel_name() const { + return kernel_name; + } + const std::string& get_operator_name() const { + return operator_name; + } + float get_avg_time_us() const; + size_t get_num_iterations() const { + return iter_timings.size(); + } + const std::vector& get_iter_timings() const { + return iter_timings; + } + CorrectnessStatus get_correctness_status() const { + return correctness_status_; + } + + // Setters + void set_kernel_name(const std::string& name) { + kernel_name = name; + } + void set_operator_name(const std::string& name) { + operator_name = name; + } + void set_correctness_status(CorrectnessStatus status) { + correctness_status_ = status; + } + + // Statistics + float get_min_time_us() const; + float get_max_time_us() const; + float get_std_dev_us() const; + + // Clear all timings + void clear_timings() { + iter_timings.clear(); + } + + // Print progress for this benchmark result + void print_summary( + int case_number, + const std::string& size_info, + float total_gflops) const; + + private: + std::string kernel_name; + std::string operator_name; + std::vector + iter_timings; // Individual iteration timings in microseconds + std::vector shader_timings_; // Per-shader timing data + CorrectnessStatus correctness_status_; +}; + +// Test result collection and processing +class TestResult { + public: + TestResult() : gflops_(0.0f), correctness_passed_(true) {} + explicit TestResult(const std::string& operation_name) + : operation_name_(operation_name), + gflops_(0.0f), + correctness_passed_(true) {} + + // Add a benchmark result + void add_result(const BenchmarkResult& result); + void add_result(BenchmarkResult&& result); + + // Getters + const std::string& get_operation_name() const { + return operation_name_; + } + float get_gflops() const { + return gflops_; + } + bool get_correctness_passed() const { + return correctness_passed_; + } + size_t size() const { + return results_.size(); + } + bool empty() const { + return results_.empty(); + } + + // Setters + void set_gflops(float gflops_val) { + gflops_ = gflops_val; + } + void set_correctness_passed(bool passed) { + correctness_passed_ = passed; + } + + // Access results + const BenchmarkResult& operator[](size_t index) const { + return results_[index]; + } + BenchmarkResult& operator[](size_t index) { + return results_[index]; + } + const std::vector& get_results() const { + return results_; + } + + // Iterator support + std::vector::iterator begin() { + return results_.begin(); + } + std::vector::iterator end() { + return results_.end(); + } + std::vector::const_iterator begin() const { + return results_.begin(); + } + std::vector::const_iterator end() const { + return results_.end(); + } + + // Processing and analysis + void print_summary() const; + void print_detailed_results() const; + void print_statistics() const; + void print_brief_summary() const; + + // Get aggregate statistics + float get_total_avg_time_us() const; + float get_total_gflops() const; + size_t get_passed_count() const; + size_t get_failed_count() const; + + // Find best/worst performing results + const BenchmarkResult* get_fastest_result() const; + const BenchmarkResult* get_slowest_result() const; + const BenchmarkResult* get_highest_gflops_result() const; + + // Clear all results + void clear() { + results_.clear(); + } + + // Set operation name + void set_operation_name(const std::string& name) { + operation_name_ = name; + } + + private: + std::string operation_name_; + std::vector results_; + float gflops_; + bool correctness_passed_; +}; + +// +// Print utilities +// + +void print_performance_header(); +void print_separator(); + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/runner.cpp b/backends/vulkan/test/ops/benchmark/framework/runner.cpp new file mode 100644 index 00000000000..35c030da58a --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/runner.cpp @@ -0,0 +1,757 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "runner.h" +#include "config.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +namespace { + +// Upper bound on the auto-sized chained_dispatches factor. Caps the factor +// when an op is fast enough that target_us / probe_us would otherwise +// exceed this; past this point, stacking more dispatches doesn't materially +// improve measurement precision. +constexpr int kMaxChainedDispatches = 200; + +// Floor for probe_us to avoid division-by-zero / a runaway chained_dispatches +// factor when an op finishes faster than the wall clock can resolve. +constexpr float kMinProbeTimeUs = 1.0f; + +} // namespace + +// Benchmark graphs are immutable after input upload, so the operator nodes can +// be recorded repeatedly. Staging uploads/downloads are encoded once: +// repeating them would redo host-visible copies on every repetition and bias +// per-invocation timings. +RepeatedGraphExecutor::RepeatedGraphExecutor( + ComputeGraph& graph, + int repetitions, + OpNodeRange op_nodes) + : graph_(graph) { + VK_CHECK_COND(repetitions > 0); + VK_CHECK_COND(op_nodes.begin <= op_nodes.end); + VK_CHECK_COND(op_nodes.end <= graph_.execute_nodes().size()); + + api::Context* const context = graph_.context(); + context->flush(); + context->set_cmd(/*reusable=*/true); + context->cmd_reset_querypool(); + + for (size_t i = 0; i < op_nodes.begin; ++i) { + graph_.execute_nodes()[i]->encode(&graph_); + } + for (int i = 0; i < repetitions; ++i) { + for (size_t j = op_nodes.begin; j < op_nodes.end; ++j) { + graph_.execute_nodes()[j]->encode(&graph_); + } + } + for (size_t i = op_nodes.end; i < graph_.execute_nodes().size(); ++i) { + graph_.execute_nodes()[i]->encode(&graph_); + } + + command_ = + std::make_unique(std::move(context->extract_cmd())); +} + +void RepeatedGraphExecutor::execute() { + api::Context* const context = graph_.context(); + command_->end(); + + // Intentionally bypasses Context::submit_cmd_to_gpu(): its submit-count + // bookkeeping and threshold-split path only serve submit_compute_job + // recording, which benchmarks don't use. + vkapi::VulkanFence fence = context->fences().get_fence(); + context->adapter_ptr()->submit_cmd( + context->queue(), + command_->get_submit_handle(/*final_use=*/false), + fence.get_submit_handle()); + fence.wait(); + context->fences().return_fence(fence); +} + +// Helper function to collect GPU timing from querypool +float collect_gpu_timing_us( + ComputeGraph& graph, + const std::vector& shader_filter) { + graph.context()->querypool().extract_results(); + const auto results = graph.context()->querypool().get_shader_timestamp_data(); + if (!results.empty()) { + // Sum durations of all shaders that don't match any pattern in + // shader_filter + float total_duration_us = 0.0f; + for (const auto& shader_result : results) { + bool filtered = false; + // Check if this shader matches any filter pattern + for (const auto& filter_pattern : shader_filter) { + if (shader_result.kernel_name.find(filter_pattern) != + std::string::npos) { + filtered = true; + break; + } + } + + if (!filtered) { + // Calculate duration from start and end times, convert from ns to μs + uint64_t duration_ns = + shader_result.end_time_ns - shader_result.start_time_ns; + total_duration_us += static_cast(duration_ns) / 1000.0f; + } + } + return total_duration_us; + } + return 0.0f; +} + +// Helper function to collect per-shader GPU timing from querypool +// Returns a map of shader_name -> timing_us for non-filtered shaders +std::unordered_map collect_per_shader_timing_us( + ComputeGraph& graph, + const std::vector& shader_filter) { + std::unordered_map shader_timings; + + graph.context()->querypool().extract_results(); + const auto results = graph.context()->querypool().get_shader_timestamp_data(); + for (const auto& shader_result : results) { + bool filtered = false; + // Check if this shader matches any filter pattern + for (const auto& filter_pattern : shader_filter) { + if (shader_result.kernel_name.find(filter_pattern) != std::string::npos) { + filtered = true; + break; + } + } + + if (!filtered) { + // Calculate duration from start and end times, convert from ns to μs + uint64_t duration_ns = + shader_result.end_time_ns - shader_result.start_time_ns; + float duration_us = static_cast(duration_ns) / 1000.0f; + // Accumulate timing for shaders with the same name + shader_timings[shader_result.kernel_name] += duration_us; + } + } + return shader_timings; +} + +// Default FLOP calculation function (assumes 1 FLOP per element) +int64_t default_flop_calculator(const TestCase& test_case) { + // Calculate total elements from the first input tensor + int64_t total_elements = 1; + if (!test_case.empty() && test_case.num_inputs() > 0 && + test_case.inputs()[0].is_tensor()) { + const auto& sizes = test_case.inputs()[0].get_tensor_sizes(); + for (int64_t size : sizes) { + total_elements *= size; + } + } + + // Assume 1 FLOP per element for basic operations + return total_elements; +} + +BenchmarkGraph setup_compute_graph( + TestCase& test_case, + const std::string& op_name, + int op_invocations_per_execute) { + GraphConfig config; + config.enable_querypool = true; + // Pool sizing takes max(execute, prepack) * factor, so scaling the factor + // also over-reserves the prepack side (encoded once). Accepted: precise + // execute-only sizing would need runtime changes. + config.descriptor_pool_safety_factor *= + std::max(1, op_invocations_per_execute); + // Default-on (opt-out via TestCase::set_force_resize(false)): force every + // DynamicDispatchNode to run its resize function when execute_test_case + // runs propagate_resize() after prepack, exercising the op's resize formula + // even when input shapes are unchanged. + config.force_resize = test_case.get_force_resize(); + config.force_narrow_int4_tile = test_case.get_force_narrow_int4_tile(); + auto graph = std::make_unique(config); + + std::vector input_values; + + // Process input ValueSpecs + for (size_t i = 0; i < test_case.num_inputs(); ++i) { + const ValueSpec& input_spec = test_case.inputs()[i]; + + if (input_spec.is_none()) { + input_values.push_back(graph->add_none()); + } else if (input_spec.is_float()) { + ValueRef input_value = + graph->add_scalar(static_cast(input_spec.get_float_value())); + input_values.push_back(input_value); + } else if (input_spec.is_int()) { + ValueRef input_value = + graph->add_scalar(static_cast(input_spec.get_int_value())); + input_values.push_back(input_value); + } else if (input_spec.is_bool()) { + ValueRef input_value = graph->add_scalar(input_spec.get_bool_value()); + input_values.push_back(input_value); + } else if (input_spec.is_int_list()) { + // Convert int32_t list to int64_t list for ComputeGraph + const auto& int32_list = input_spec.get_int_list(); + std::vector int64_list; + int64_list.reserve(int32_list.size()); + for (int32_t val : int32_list) { + int64_list.push_back(static_cast(val)); + } + ValueRef input_value = graph->add_scalar_list(std::move(int64_list)); + input_values.push_back(input_value); + } else if (input_spec.is_string()) { + std::string str_copy = input_spec.get_string_value(); + ValueRef input_value = graph->add_string(std::move(str_copy)); + input_values.push_back(input_value); + } else if (input_spec.is_constant()) { + ValueRef input_value = graph->add_tensorref( + input_spec.get_tensor_sizes(), + input_spec.dtype, + input_spec.get_data_ptr()); + input_values.push_back(input_value); + } else { + IOValueRef input_io = graph->add_input_tensor( + input_spec.get_tensor_sizes(), + input_spec.dtype, + input_spec.storage_type, + input_spec.memory_layout); + input_values.push_back(input_io.value); + } + } + + std::vector output_values; + + // Process output ValueSpecs + for (size_t i = 0; i < test_case.num_outputs(); ++i) { + const ValueSpec& output_spec = test_case.outputs()[i]; + + if (!output_spec.is_tensor()) { + throw std::runtime_error("All output specifications must be tensors"); + } + + // Create output tensor + ValueRef output_value = graph->add_tensor( + output_spec.get_tensor_sizes(), + output_spec.dtype, + output_spec.storage_type, + output_spec.memory_layout); + + output_values.push_back(output_value); + } + + // Get the operator function and call it + auto opFn = VK_GET_OP_FN(op_name); + + // Create arguments vector for the operator function + std::vector op_args = input_values; + op_args.insert(op_args.end(), output_values.begin(), output_values.end()); + + // Nodes added before the op are staging uploads; nodes added after are + // staging downloads. Only the op's own nodes are repeated by benchmarks. + const size_t op_begin = graph->execute_nodes().size(); + opFn(*graph, op_args); + const size_t op_end = graph->execute_nodes().size(); + + for (size_t i = 0; i < output_values.size(); ++i) { + graph->set_output_value(output_values[i]); + } + return {std::move(graph), {op_begin, op_end}}; +} + +// Test execution utilities +BenchmarkResult execute_test_case( + TestCase& test_case, + int warmup_runs, + int benchmark_runs, + int chained_dispatches, + bool write_outputs) { + BenchmarkResult result( + test_case.name().empty() ? "unnamed_test_case" : test_case.name()); + + // Initialize querypool if using GPU timestamps + if (use_gpu_timestamps()) { + api::context()->initialize_querypool(); + } + + // Build the operator once. Benchmark repetition is encoded separately so + // persistent graph allocations are not duplicated. + BenchmarkGraph benchmark = setup_compute_graph( + test_case, test_case.operator_name(), chained_dispatches); + ComputeGraph& graph = *benchmark.graph; + + // Prepare the graph + graph.prepare(); + graph.prepack(); + + // Run resize functions once so force_resize exercises resize formulas even + // though the record/replay path below never calls propagate_resize(). + graph.propagate_resize(); + + // Copy input data into the graph's staging buffers + size_t graph_input_idx = 0; + for (size_t i = 0; i < test_case.num_inputs(); ++i) { + const ValueSpec& input_spec = test_case.inputs()[i]; + + // Only non-constant tensor inputs correspond to graph.inputs() entries + bool is_graph_input = input_spec.is_tensor() && !input_spec.is_constant() && + !input_spec.is_none(); + if (!is_graph_input) { + continue; + } + + if (graph_input_idx < graph.inputs().size()) { + const auto& input_ref = graph.inputs()[graph_input_idx]; + + // Get the appropriate data based on dtype + const void* data_ptr = nullptr; + size_t data_numel = input_spec.numel(); + + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (input_spec.dtype) { + case vkapi::kFloat: + data_ptr = input_spec.get_float_data().data(); + break; + case vkapi::kHalf: + data_ptr = input_spec.get_half_data().data(); + break; + case vkapi::kInt: + data_ptr = input_spec.get_int32_data().data(); + break; + case vkapi::kChar: + data_ptr = input_spec.get_int8_data().data(); + break; + case vkapi::kByte: + data_ptr = input_spec.get_uint8_data().data(); + break; + default: + throw std::runtime_error("Unsupported data type for input tensor"); + } + + // Copy data into staging buffer + graph.maybe_cast_and_copy_into_staging( + input_ref.staging, data_ptr, data_numel, input_spec.dtype); + } + ++graph_input_idx; + } + + RepeatedGraphExecutor graph_executor( + graph, chained_dispatches, benchmark.op_nodes); + + // Warmup runs + for (int run = 0; run < warmup_runs; ++run) { + graph_executor.execute(); + } + + // Benchmark runs - collect individual iteration timings + float total_cpu_time_us = 0.0f; + float total_gpu_time_us = 0.0f; + + const float chained_dispatches_f = static_cast(chained_dispatches); + + for (int run = 0; run < benchmark_runs; ++run) { + // Measure CPU time for each execute() call + auto cpu_start = std::chrono::high_resolution_clock::now(); + graph_executor.execute(); + auto cpu_end = std::chrono::high_resolution_clock::now(); + + auto cpu_duration = std::chrono::duration_cast( + cpu_end - cpu_start); + // Each execute() ran chained_dispatches op invocations; report + // per-invocation time. + float cpu_time_us = + static_cast(cpu_duration.count()) / chained_dispatches_f; + total_cpu_time_us += cpu_time_us; + + // Collect per-shader GPU timing - get raw shader results to preserve + // metadata + graph.context()->querypool().extract_results(); + const auto shader_results = + graph.context()->querypool().get_shader_timestamp_data(); + + // Calculate total GPU time from per-shader timings + float gpu_time_us = 0.0f; + for (const auto& shader_result : shader_results) { + // Check if this shader matches any filter pattern + bool filtered = false; + for (const auto& filter_pattern : test_case.get_shader_filter()) { + if (shader_result.kernel_name.find(filter_pattern) != + std::string::npos) { + filtered = true; + break; + } + } + + if (!filtered) { + uint64_t duration_ns = + shader_result.end_time_ns - shader_result.start_time_ns; + float duration_us = static_cast(duration_ns) / 1000.0f; + gpu_time_us += duration_us; + // Per-shader sample is already one querypool entry per dispatch (at + // chained_dispatches>1 we get that many entries per shader), so do NOT + // divide here — the aggregator (get_avg_time_us) averages across them + // naturally. + result.add_shader_timing( + shader_result.kernel_name, + duration_us, + shader_result.metadata.gwg, + shader_result.metadata.lwg); + } + } + // gpu_time_us aggregates chained_dispatches worth of shader runs; divide + // it down to per-invocation time. + gpu_time_us /= chained_dispatches_f; + total_gpu_time_us += gpu_time_us; + + // Add the appropriate timing based on the flag + float iter_time_us = use_gpu_timestamps() ? gpu_time_us : cpu_time_us; + result.add_iter_timing(iter_time_us); + } + + // Calculate averages for display + float avg_cpu_time_us = total_cpu_time_us / benchmark_runs; + float avg_gpu_time_us = total_gpu_time_us / benchmark_runs; + + // Print both timings if latency printing is enabled + if (print_latencies()) { + if (use_gpu_timestamps()) { + graph.context()->querypool().print_results(); + } + std::cout << " CPU timing: " << std::fixed << std::setprecision(3) + << avg_cpu_time_us << " μs" << std::endl; + std::cout << " GPU timing: " << std::fixed << std::setprecision(3) + << avg_gpu_time_us << " μs" << std::endl; + std::cout << " Using " << (use_gpu_timestamps() ? "GPU" : "CPU") + << " timing for result" << std::endl; + } + + // Copy output data from the graph's staging buffers. The benchmarking path + // skips this work since it doesn't need correctness data — the probe pass + // has already populated test_case.outputs() at chained_dispatches=1. + if (write_outputs) { + for (size_t i = 0; i < test_case.num_outputs(); ++i) { + ValueSpec& output_spec = test_case.outputs()[i]; + + if (output_spec.is_tensor() && i < graph.outputs().size()) { + const auto& output_ref = graph.outputs()[i]; + + // Ensure output data vector is properly sized + size_t data_numel = output_spec.numel(); + output_spec.resize_data(data_numel); + + // Get mutable data pointer for the output + void* data_ptr = output_spec.get_mutable_data_ptr(); + + if (data_ptr != nullptr) { + // Copy data from staging buffer to output spec + graph.maybe_cast_and_copy_from_staging( + output_ref.staging, data_ptr, data_numel, output_spec.dtype); + } + + // Print output tensor data if output printing is enabled + if (print_output()) { + std::string output_name = "Output[" + std::to_string(i) + "]"; + print_valuespec_data(output_spec, output_name); + } + } + } + } + + return result; +} + +TestResult execute_test_cases( + std::vector test_cases, + const FlopCalculatorFunc& flop_calculator, + const std::string& operation_name, + int warmup_runs, + int benchmark_runs, + const ReferenceComputeFunc& reference_compute_func) { + TestResult results(operation_name); + + std::cout << "Executing " << test_cases.size() << " test cases for " + << operation_name << std::endl; + print_separator(); + + // Group test cases by ReferenceKey for caching reference computations + // Use a vector to preserve the order in which groups first appear + std::vector group_order; + std::unordered_map, ReferenceKeyHash> + groups; + for (size_t i = 0; i < test_cases.size(); ++i) { + ReferenceKey key = ReferenceKey::from_test_case(test_cases[i]); + if (groups.find(key) == groups.end()) { + group_order.push_back(key); + } + groups[key].push_back(i); + } + + bool any_correctness_failed = false; + float total_gflops = 0.0f; + size_t test_case_counter = 0; + + // Process each group: generate data, compute reference, execute, and print + // Iterate in the order groups first appeared in test_cases + for (const auto& key : group_order) { + const auto& indices = groups[key]; + if (indices.empty()) { + continue; + } + + // Get first test case as the "prototype" + size_t prototype_idx = indices[0]; + TestCase& prototype = test_cases[prototype_idx]; + + // Generate data for prototype with deterministic seed based on key + int group_seed = + static_cast(std::hash{}(key.key_string) % 10000); + for (auto& input : prototype.inputs()) { + input.ensure_data_generated(group_seed++); + } + + // Compute reference once for prototype + bool ref_computed = false; + if (reference_compute_func) { + try { + reference_compute_func(prototype); + ref_computed = true; + } catch (const std::invalid_argument&) { + // Reference computation skipped for this group + } + } + + // Copy data and reference to other test cases in group + for (size_t i = 1; i < indices.size(); ++i) { + size_t tc_idx = indices[i]; + TestCase& tc = test_cases[tc_idx]; + + // Copy input data from prototype + for (size_t j = 0; + j < tc.inputs().size() && j < prototype.inputs().size(); + ++j) { + auto& dest = tc.inputs()[j]; + const auto& src = prototype.inputs()[j]; + if (dest.is_tensor() && src.is_tensor() && dest.sizes == src.sizes && + dest.dtype == src.dtype) { + dest.share_data_from(src); + } + } + + // Copy reference output data if available + if (ref_computed) { + for (size_t j = 0; + j < tc.outputs().size() && j < prototype.outputs().size(); + ++j) { + const auto& src = prototype.outputs()[j]; + auto& dest = tc.outputs()[j]; + if (dest.is_tensor() && src.is_tensor() && dest.sizes == src.sizes && + dest.dtype == src.dtype) { + dest.share_reference_from(src); + } + } + } + } + + // Execute and print results for all test cases in this group + for (size_t tc_idx : indices) { + TestCase& test_case = test_cases[tc_idx]; + ++test_case_counter; + + // Execute single test case + BenchmarkResult result; + bool shader_not_supported = false; + try { + // Always run a probe pass at chained_dispatches=1 with + // write_outputs=true. This populates test_case.outputs() for the + // downstream correctness check with a clean single-dispatch result, + // and also gives us probe_us for adaptive sizing of the measurement + // run's chained_dispatches. + // + // probe_then_scale (Google Benchmark style): size chained_dispatches + // so each measurement execute() takes ~target_us. Tiny ops get a + // large factor (driving GPU governor escalation); heavy ops get a + // small factor (bounded wall time). Caveat: on Adreno 740 the probe + // runs at the DCVS-pinned clock (~220 MHz), so probe_us is inflated + // and the computed factor comes out under-sized for boost clock — but + // the default target is generous enough that an under-sized factor + // still drives sustained activity. + BenchmarkResult probe_result = execute_test_case( + test_case, + /*warmup_runs=*/1, + /*benchmark_runs=*/1, + /*chained_dispatches=*/1, + /*write_outputs=*/true); + float probe_us = probe_result.get_avg_time_us(); + if (probe_us < kMinProbeTimeUs) { + probe_us = kMinProbeTimeUs; + } + + int chained_dispatches; + const int manual_n = test_case.get_op_invocations_per_execute(); + if (manual_n > 0) { + chained_dispatches = manual_n; + } else { + const int target_us = test_case.get_target_execute_time_us(); + chained_dispatches = + std::max(1, static_cast(target_us / probe_us)); + chained_dispatches = + std::min(chained_dispatches, kMaxChainedDispatches); + } + if (debugging()) { + std::cout << "[probe] " << test_case.name() + << ": probe_us=" << probe_us + << ", chained_dispatches=" << chained_dispatches + << std::endl; + } + + // Measurement pass: chained_dispatches stacking, skip output copy + // since the probe already wrote the correctness data. + result = execute_test_case( + test_case, + warmup_runs, + benchmark_runs, + chained_dispatches, + /*write_outputs=*/false); + result.set_operator_name(test_case.operator_name()); + } catch (const vkcompute::vkapi::ShaderNotSupportedError&) { + result = BenchmarkResult( + test_case.name().empty() ? "unnamed_test_case" : test_case.name(), + test_case.operator_name()); + shader_not_supported = true; + } + + // Determine if this test case passed (has valid timing data) + bool vulkan_execute_succeeded = + result.get_num_iterations() > 0 && result.get_avg_time_us() > 0.0f; + + if (shader_not_supported) { + result.set_correctness_status(CorrectnessStatus::SKIPPED); + } else if (!vulkan_execute_succeeded) { + result.set_correctness_status(CorrectnessStatus::FAILED); + } else if (!ref_computed) { + result.set_correctness_status(CorrectnessStatus::SKIPPED); + } else { + // Reference function provided and succeeded - validate outputs + bool correctness_passed = true; + + for (size_t output_idx = 0; output_idx < test_case.num_outputs(); + ++output_idx) { + const ValueSpec& output_spec = test_case.outputs()[output_idx]; + + if (!output_spec.validate_against_reference( + test_case.get_abs_tolerance(), + test_case.get_rel_tolerance())) { + correctness_passed = false; + std::cout << " Correctness validation FAILED for test " + << result.get_kernel_name() << std::endl; + print_valuespec_data(output_spec, "vulkan output"); + print_valuespec_data(output_spec, "ref output", true); + } + } + + if (correctness_passed) { + result.set_correctness_status(CorrectnessStatus::PASSED); + } else { + any_correctness_failed = true; + result.set_correctness_status(CorrectnessStatus::FAILED); + } + } + + // Calculate GFLOPS for this test case using the provided FLOP calculator + float case_gflops = 0.0f; + if (vulkan_execute_succeeded) { + // Use the provided FLOP calculator to get total FLOPs for this test + // case + int64_t total_flops = flop_calculator(test_case); + float flops = static_cast(total_flops); + float avg_time_us = result.get_avg_time_us(); + if (avg_time_us > 0.0f && total_flops > 0) { + case_gflops = (flops / 1e9f) / (avg_time_us / 1e6f); + } + + total_gflops += case_gflops; + } else { + case_gflops = -1.0f; // Indicate failure + } + + // Calculate tensor info for display + std::string size_info = "["; + if (!test_case.empty() && test_case.num_inputs() > 0 && + test_case.inputs()[0].is_tensor()) { + const auto& sizes = test_case.inputs()[0].get_tensor_sizes(); + for (size_t j = 0; j < sizes.size(); ++j) { + size_info += std::to_string(sizes[j]); + if (j < sizes.size() - 1) { + size_info += "x"; + } + } + } + size_info += "]"; + + // Print progress using the BenchmarkResult member function + result.print_summary( + utils::safe_downcast(test_case_counter), size_info, case_gflops); + + // Add result to collection + results.add_result(std::move(result)); + + test_case.clear(); + } + } + + // Set the overall results on the TestResult + results.set_correctness_passed(!any_correctness_failed); + results.set_gflops(total_gflops); + + print_separator(); + std::cout << "Completed " << results.size() << " test cases" << std::endl; + + // Hard-fail if any correctness check failed. The per-case loop above keeps + // running after a mismatch (so the full pass/fail matrix and per-16x16-tile + // mismatch maps are printed for debugging) rather than aborting on the first + // failure; throwing here preserves the original abort-on-mismatch behavior. + if (any_correctness_failed) { + throw std::runtime_error("Correctness validation failed"); + } + + return results; +} + +// Convenience overload that uses the default FLOP calculator +TestResult execute_test_cases( + std::vector test_cases, + const std::string& operation_name, + int warmup_runs, + int benchmark_runs, + const ReferenceComputeFunc& reference_compute_func) { + return execute_test_cases( + std::move(test_cases), + default_flop_calculator, + operation_name, + warmup_runs, + benchmark_runs, + reference_compute_func); +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/runner.h b/backends/vulkan/test/ops/benchmark/framework/runner.h new file mode 100644 index 00000000000..56a7462b3c6 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/runner.h @@ -0,0 +1,123 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include "results.h" +#include "test_case.h" + +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// +// Test case execution +// + +using FlopCalculatorFunc = std::function; + +// Default FLOP calculation function (assumes 1 FLOP per element) +int64_t default_flop_calculator(const TestCase& test_case); + +using ReferenceComputeFunc = std::function; + +// Half-open index range of the operator's own dispatch nodes within a +// benchmark graph's execute_nodes(). Staging upload nodes precede it, staging +// download nodes follow it. +struct OpNodeRange { + size_t begin = 0; + size_t end = 0; +}; + +// A benchmark graph plus the location of its repeatable operator nodes. The +// graph is heap-held: ComputeGraph owns its Context and must never be moved +// (a moved-from graph's destructor dereferences a null context). +struct BenchmarkGraph { + std::unique_ptr graph; + OpNodeRange op_nodes; +}; + +// Benchmark-only executor that records a graph's execute nodes into a single +// reusable command buffer, then replays it on every execute(). Staging uploads +// are encoded once, operator nodes N times, staging downloads once, so each +// iteration performs the same work as the old stacked-nodes layout (1 upload + +// N ops + 1 download) and the per-invocation divisor is unchanged. Production +// ComputeGraph::execute() behavior is unchanged. +// +// Notes for interpreting benchmark numbers: +// - Submit granularity differs from stacking N distinct nodes: all encodings +// live in one command buffer with one submit per iteration (the old path +// could split across command buffers at the node-count threshold), so +// per-dispatch times may shift systematically against older data. +// - Resize functions run once via propagate_resize() in execute_test_case +// before recording; replay itself never re-triggers resize. +// - Repeated encodings share one node/dispatch id, so per-repetition +// querypool attribution is unavailable (aggregation keys on kernel name). +class RepeatedGraphExecutor final { + public: + RepeatedGraphExecutor( + ComputeGraph& graph, + int repetitions, + OpNodeRange op_nodes); + void execute(); + + private: + ComputeGraph& graph_; + std::unique_ptr command_; +}; + +// Runs a measurement at the given chained_dispatches factor. The operator is +// built once, then its execute nodes are encoded that many times into a +// benchmark-only reusable command buffer. The probe-then-scale orchestration +// lives in execute_test_cases(). +// +// write_outputs controls whether the graph's staging output buffers are copied +// back into test_case.outputs() at the end of the run. The probe path needs +// write_outputs=true so the correctness check has a clean single-dispatch +// reference. The benchmarking path passes write_outputs=false to avoid the +// per-iter GPU->CPU copy cost. +BenchmarkResult execute_test_case( + TestCase& test_case, + int warmup_runs = 1, + int benchmark_runs = 1, + int chained_dispatches = 1, + bool write_outputs = true); + +TestResult execute_test_cases( + std::vector test_cases, + const FlopCalculatorFunc& flop_calculator, + const std::string& operation_name = "Operation", + int warmup_runs = 1, + int benchmark_runs = 1, + const ReferenceComputeFunc& reference_compute_func = nullptr); + +TestResult execute_test_cases( + std::vector test_cases, + const std::string& operation_name = "Operation", + int warmup_runs = 1, + int benchmark_runs = 1, + const ReferenceComputeFunc& reference_compute_func = nullptr); + +// Setup compute graph based on TestCase and operation name. The op function is +// invoked once. op_invocations_per_execute is used only to reserve enough +// descriptor capacity for benchmark-only repeated command encoding. Returns +// the graph plus the range of the operator's own nodes for repeated encoding. +BenchmarkGraph setup_compute_graph( + TestCase& test_case, + const std::string& op_name, + int op_invocations_per_execute = 1); + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/test_case.cpp b/backends/vulkan/test/ops/benchmark/framework/test_case.cpp new file mode 100644 index 00000000000..550bd10c136 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/test_case.cpp @@ -0,0 +1,79 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "test_case.h" + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// ReferenceKey implementation +ReferenceKey ReferenceKey::from_test_case(const TestCase& tc) { + std::ostringstream oss; + + // Serialize inputs that affect reference computation + // Skip: storage_type, memory_layout, string values (like impl_selector) + for (size_t i = 0; i < tc.inputs().size(); ++i) { + const ValueSpec& input = tc.inputs()[i]; + oss << "i" << i << ":"; + + if (input.is_tensor()) { + // For tensors: sizes, dtype, data_gen_type, is_constant + oss << "T["; + for (size_t j = 0; j < input.sizes.size(); ++j) { + if (j > 0) { + oss << ","; + } + oss << input.sizes[j]; + } + oss << "]d" << static_cast(input.dtype); + oss << "g" << static_cast(input.data_gen_type); + oss << "c" << (input.is_constant() ? 1 : 0); + oss << "n" << (input.is_none() ? 1 : 0); + } else if (input.is_int()) { + oss << "I" << input.get_int_value(); + } else if (input.is_float()) { + oss << "F" << input.get_float_value(); + } else if (input.is_bool()) { + oss << "B" << (input.get_bool_value() ? 1 : 0); + } else if (input.is_int_list()) { + oss << "L["; + const auto& list = input.get_int_list(); + for (size_t j = 0; j < list.size(); ++j) { + if (j > 0) { + oss << ","; + } + oss << list[j]; + } + oss << "]"; + } + // Skip string inputs (like impl_selector) as they don't affect reference + oss << ";"; + } + + // Also include output shapes for completeness + for (size_t i = 0; i < tc.outputs().size(); ++i) { + const ValueSpec& output = tc.outputs()[i]; + oss << "o" << i << ":["; + for (size_t j = 0; j < output.sizes.size(); ++j) { + if (j > 0) { + oss << ","; + } + oss << output.sizes[j]; + } + oss << "]d" << static_cast(output.dtype) << ";"; + } + + ReferenceKey key; + key.key_string = oss.str(); + return key; +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/test_case.h b/backends/vulkan/test/ops/benchmark/framework/test_case.h new file mode 100644 index 00000000000..5edd5838ffc --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/test_case.h @@ -0,0 +1,236 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include "value_spec.h" + +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +// +// ReferenceKey for caching reference computations +// + +// Captures the identity of input conditions for test case grouping. +// Test cases with the same ReferenceKey should produce identical reference +// outputs, so reference computation can be cached and reused. +struct ReferenceKey { + std::string key_string; + + static ReferenceKey from_test_case(const class TestCase& tc); + + bool operator==(const ReferenceKey& other) const { + return key_string == other.key_string; + } +}; + +struct ReferenceKeyHash { + size_t operator()(const ReferenceKey& k) const { + return std::hash{}(k.key_string); + } +}; + +// +// Shader filter defaults +// + +// Default shader filter: excludes layout conversions and quantization overhead +// This is used for tests where quantize/dequantize are overhead operations +inline const std::vector kDefaultShaderFilter = { + "nchw_to", + "to_nchw", + "quantize_and_pack_4w4c", + "unpack_4w4c_and_dequantize"}; + +// Layout-only filter: only excludes layout conversions +// Use this for tests where quantize/dequantize ARE the operations being tested +inline const std::vector kLayoutOnlyShaderFilter = { + "nchw_to", + "to_nchw"}; + +// +// TestCase +// + +// Default per-execute() wall-clock target used by the probe-then-scale logic +// in execute_test_cases(). Picked generously enough that even an under-sized +// chained_dispatches factor (the probe runs at governor-pinned clock and +// underestimates the boost-clock latency) still drives sustained GPU +// activity during measurement. +constexpr int kDefaultTargetExecuteTimeUs = 100000; + +class TestCase { + public: + TestCase() + : abs_tolerance_(2e-3f), + rel_tolerance_(1e-3f), + shader_filter_(kDefaultShaderFilter) {} + explicit TestCase(const std::string& name) + : name_(name), + abs_tolerance_(2e-3f), + rel_tolerance_(1e-3f), + shader_filter_(kDefaultShaderFilter) {} + + void set_name(const std::string& name) { + name_ = name; + } + const std::string& name() const { + return name_; + } + + void set_operator_name(const std::string& op_name) { + operator_name_ = op_name; + } + const std::string& operator_name() const { + return operator_name_; + } + + // Tolerance settings + void set_abs_tolerance(float abs_tolerance) { + abs_tolerance_ = abs_tolerance; + } + float get_abs_tolerance() const { + return abs_tolerance_; + } + + void set_rel_tolerance(float rel_tolerance) { + rel_tolerance_ = rel_tolerance; + } + float get_rel_tolerance() const { + return rel_tolerance_; + } + + // Shader filter settings - list of shader name patterns to exclude from + // timing + void set_shader_filter(const std::vector& filter) { + shader_filter_ = filter; + } + const std::vector& get_shader_filter() const { + return shader_filter_; + } + + // Manual override for the number of chained dispatches per measurement + // iteration (a.k.a. chained_dispatches). If > 0, the framework uses this + // directly and skips the probe phase. 0 (the default) means adaptive + // (probe-then-scale). + void set_op_invocations_per_execute(int n) { + op_invocations_per_execute_ = n; + } + int get_op_invocations_per_execute() const { + return op_invocations_per_execute_; + } + + // Target single-execute duration in microseconds. Used only when the + // manual chained_dispatches override is not set. Default + // kDefaultTargetExecuteTimeUs, picked generously enough to mitigate Adreno + // DCVS governor pinning during the probe. + void set_target_execute_time_us(int us) { + target_execute_time_us_ = us; + } + int get_target_execute_time_us() const { + return target_execute_time_us_; + } + + // When true, the ComputeGraph built for this test case sets + // GraphConfig::force_resize, so every DynamicDispatchNode runs its resize + // function once during measurement setup (execute_test_case runs + // propagate_resize() after prepack) even when no input shape changed. + // Because the output is already allocated at the swept shape, the resize + // must recompute the same shape from the current input — a wrong resize + // formula resizes the output to a mismatched shape and surfaces as a test + // failure. Default true (opt-out): every custom_ops test exercises its + // resize formulas across the swept shapes. Call set_force_resize(false) for + // the rare op whose resize fn is intentionally not shape-preserving under a + // fixed output allocation. + void set_force_resize(bool force_resize) { + force_resize_ = force_resize; + } + bool get_force_resize() const { + return force_resize_; + } + + // When true, the ComputeGraph sets GraphConfig::force_narrow_int4_tile, so + // the int4 linear picks its narrow (2-row) output tile on any GPU. That + // variant is otherwise reachable only on Mali, which no CI runner has. + void set_force_narrow_int4_tile(bool force) { + force_narrow_int4_tile_ = force; + } + bool get_force_narrow_int4_tile() const { + return force_narrow_int4_tile_; + } + + void add_input_spec(const ValueSpec& spec) { + inputs_.push_back(spec); + } + + const std::vector& inputs() const { + return inputs_; + } + + std::vector& inputs() { + return inputs_; + } + + size_t num_inputs() const { + return inputs_.size(); + } + + void add_output_spec(const ValueSpec& spec) { + outputs_.push_back(spec); + } + + const std::vector& outputs() const { + return outputs_; + } + + std::vector& outputs() { + return outputs_; + } + + size_t num_outputs() const { + return outputs_.size(); + } + + bool empty() const { + return inputs_.empty() && outputs_.empty(); + } + void clear() { + inputs_.clear(); + outputs_.clear(); + name_.clear(); + operator_name_.clear(); + abs_tolerance_ = 2e-3f; + rel_tolerance_ = 1e-3f; + shader_filter_ = kDefaultShaderFilter; + op_invocations_per_execute_ = 0; + target_execute_time_us_ = kDefaultTargetExecuteTimeUs; + force_resize_ = true; + force_narrow_int4_tile_ = false; + } + + private: + std::string name_; + std::string operator_name_; + std::vector inputs_; + std::vector outputs_; + float abs_tolerance_; + float rel_tolerance_; + std::vector shader_filter_; + int op_invocations_per_execute_ = 0; // 0 = adaptive + int target_execute_time_us_ = kDefaultTargetExecuteTimeUs; + bool force_resize_ = true; + bool force_narrow_int4_tile_ = false; +}; + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/utils.h b/backends/vulkan/test/ops/benchmark/framework/utils.h new file mode 100644 index 00000000000..51d5fd1bd8f --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/utils.h @@ -0,0 +1,27 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "labels.h" +#include "results.h" +#include "runner.h" +#include "test_case.h" +#include "value_spec.h" +#include "weight_utils.h" diff --git a/backends/vulkan/test/ops/benchmark/framework/value_spec.cpp b/backends/vulkan/test/ops/benchmark/framework/value_spec.cpp new file mode 100644 index 00000000000..b9752596f89 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/value_spec.cpp @@ -0,0 +1,1024 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "value_spec.h" +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +int get_seed() { + static int seed = 42; + return seed++; +} + +int get_seed_or_explicit(int explicit_seed) { + if (explicit_seed >= 0) { + return explicit_seed; + } + return get_seed(); +} + +// Forward declarations for data generation utilities +void generate_random_float_data( + std::vector& data, + float min_val = -1.0f, + float max_val = 1.0f, + int explicit_seed = -1); +void generate_random_int_data( + std::vector& data, + int min_val = -10, + int max_val = 10, + int explicit_seed = -1); +void generate_randint_float_data( + std::vector& data, + int min_val = -10, + int max_val = 10, + int explicit_seed = -1); +void generate_randint_half_data( + std::vector& data, + int min_val = -10, + int max_val = 10, + int explicit_seed = -1); +void generate_random_int8_data( + std::vector& data, + int8_t min_val = -10, + int8_t max_val = 10, + int explicit_seed = -1); +void generate_random_uint8_data( + std::vector& data, + uint8_t min_val = 0, + uint8_t max_val = 255, + int explicit_seed = -1); +void generate_random_2xint4_data( + std::vector& data, + int explicit_seed = -1); +void generate_random_2xint4_data( + std::vector& data, + int explicit_seed = -1); +void generate_random_int4_data( + std::vector& data, + int8_t min_val = -8, + int8_t max_val = 7, + int explicit_seed = -1); +void generate_ones_data(std::vector& data); +void generate_zeros_data(std::vector& data); + +// Convert a float32 value to IEEE 754 half-precision (uint16_t) +uint16_t float_to_half(float value) { + uint32_t float_bits; + std::memcpy(&float_bits, &value, sizeof(float)); + uint32_t sign = (float_bits >> 31) & 0x1; + int32_t exponent = static_cast((float_bits >> 23) & 0xFF) - 127; + uint32_t mantissa = float_bits & 0x7FFFFF; + + uint16_t half_val; + if (exponent > 15) { + half_val = static_cast((sign << 15) | 0x7C00); // Inf + } else if (exponent < -14) { + half_val = static_cast(sign << 15); // Zero / subnormal + } else { + half_val = static_cast( + (sign << 15) | (static_cast(exponent + 15) << 10) | + (mantissa >> 13)); + } + return half_val; +} + +// Convert a IEEE 754 half-precision (uint16_t) value to float32 +float half_to_float(uint16_t half_val) { + uint32_t sign = (half_val >> 15) & 0x1; + uint32_t exponent = (half_val >> 10) & 0x1F; + uint32_t mantissa = half_val & 0x3FF; + + float result; + if (exponent == 0) { + result = std::ldexp(static_cast(mantissa), -24); + } else if (exponent == 31) { + result = mantissa ? std::numeric_limits::quiet_NaN() + : std::numeric_limits::infinity(); + } else { + result = std::ldexp( + 1.0f + static_cast(mantissa) / 1024.0f, exponent - 15); + } + if (sign) { + result = -result; + } + return result; +} + +// ValueSpec implementation +void ValueSpec::ensure_unique_data() const { + if (data_.use_count() != 1) { + data_ = std::make_shared(*data_); + } +} + +void ValueSpec::ensure_unique_reference_data() const { + if (reference_data_.use_count() != 1) { + reference_data_ = std::make_shared(*reference_data_); + } +} + +void ValueSpec::generate_tensor_data(int seed) const { + if (spec_type != SpecType::Tensor) { + return; + } + + ensure_unique_data(); + auto& float_data = data_->float_data; + auto& int32_data = data_->int32_data; + auto& half_data = data_->half_data; + auto& int8_data = data_->int8_data; + auto& uint8_data = data_->uint8_data; + + int64_t num_elements = numel(); + + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: { + float_data.resize(num_elements); + if (data_gen_type == DataGenType::RANDOM) { + generate_random_float_data(float_data, -1.0f, 1.0f, seed); + } else if (data_gen_type == DataGenType::RANDOM_SCALES) { + generate_random_float_data(float_data, 0.005, 0.015, seed); + } else if (data_gen_type == DataGenType::RANDINT) { + generate_randint_float_data(float_data, -10, 10, seed); + } else if (data_gen_type == DataGenType::RANDINT8) { + generate_randint_float_data(float_data, -128, 127, seed); + } else if (data_gen_type == DataGenType::RANDINT4) { + generate_randint_float_data(float_data, -8, 7, seed); + } else if (data_gen_type == DataGenType::ONES) { + generate_ones_data(float_data); + } else if (data_gen_type == DataGenType::ZEROS) { + generate_zeros_data(float_data); + } else { + generate_zeros_data(float_data); + } + break; + } + case vkapi::kHalf: { + half_data.resize(num_elements); + if (data_gen_type == DataGenType::RANDOM) { + // Generate random float data first, then convert to IEEE 754 half. + std::vector temp_data(num_elements); + generate_random_float_data(temp_data, -1.0f, 1.0f, seed); + for (size_t i = 0; i < temp_data.size(); ++i) { + half_data[i] = float_to_half(temp_data[i]); + } + } else if (data_gen_type == DataGenType::RANDOM_SCALES) { + // Generate random scales in float, then convert to proper fp16 + std::vector temp_data(num_elements); + generate_random_float_data(temp_data, 0.005f, 0.015f, seed); + for (size_t i = 0; i < temp_data.size(); ++i) { + half_data[i] = float_to_half(temp_data[i]); + } + } else if (data_gen_type == DataGenType::RANDINT) { + generate_randint_half_data(half_data, -10, 10, seed); + } else if (data_gen_type == DataGenType::RANDINT8) { + generate_randint_half_data(half_data, -128, 127, seed); + } else if (data_gen_type == DataGenType::RANDINT4) { + generate_randint_half_data(half_data, -8, 7, seed); + } else if (data_gen_type == DataGenType::ONES) { + std::fill(half_data.begin(), half_data.end(), float_to_half(1.0f)); + } else if (data_gen_type == DataGenType::ZEROS) { + std::fill( + half_data.begin(), + half_data.end(), + static_cast(0)); // 0.0 in half + } else { + std::fill( + half_data.begin(), + half_data.end(), + static_cast(0)); // 0.0 in half + } + break; + } + case vkapi::kInt: { + int32_data.resize(num_elements); + if (data_gen_type == DataGenType::RANDOM) { + generate_random_int_data(int32_data, -10, 10, seed); + } else if (data_gen_type == DataGenType::RANDINT) { + generate_random_int_data( + int32_data, + -10, + 10, + seed); // For int type, RANDINT is same as RANDOM + } else if (data_gen_type == DataGenType::RANDINT8) { + generate_random_int_data(int32_data, -128, 127, seed); + } else if (data_gen_type == DataGenType::RANDINT4) { + generate_random_int_data(int32_data, -8, 7, seed); + } else if (data_gen_type == DataGenType::ONES) { + std::fill(int32_data.begin(), int32_data.end(), 1); + } else if (data_gen_type == DataGenType::ZEROS) { + std::fill(int32_data.begin(), int32_data.end(), 0); + } else { + std::fill(int32_data.begin(), int32_data.end(), 0); + } + break; + } + case vkapi::kChar: { + int8_data.resize(num_elements); + if (data_gen_type == DataGenType::RANDOM) { + generate_random_int8_data(int8_data, -10, 10, seed); + } else if (data_gen_type == DataGenType::RANDINT) { + generate_random_int8_data(int8_data, -10, 10, seed); + } else if (data_gen_type == DataGenType::RANDINT8) { + generate_random_int8_data(int8_data, -128, 127, seed); + } else if (data_gen_type == DataGenType::RANDINT4) { + generate_random_2xint4_data(int8_data, seed); + } else if (data_gen_type == DataGenType::ONES) { + std::fill(int8_data.begin(), int8_data.end(), 1); + } else if (data_gen_type == DataGenType::ONES_INT4) { + int8_t packed_data = (1 << 4) | 1; + std::fill(int8_data.begin(), int8_data.end(), packed_data); + } else if (data_gen_type == DataGenType::ZEROS) { + std::fill(int8_data.begin(), int8_data.end(), 0); + } else { + std::fill(int8_data.begin(), int8_data.end(), 0); + } + break; + } + case vkapi::kByte: { + uint8_data.resize(num_elements); + if (data_gen_type == DataGenType::RANDOM) { + generate_random_uint8_data(uint8_data, 0, 255, seed); + } else if (data_gen_type == DataGenType::RANDINT) { + generate_random_uint8_data(uint8_data, 0, 255, seed); + } else if (data_gen_type == DataGenType::RANDINT8) { + generate_random_uint8_data(uint8_data, 0, 255, seed); + } else if (data_gen_type == DataGenType::RANDINT4) { + generate_random_2xint4_data(uint8_data, seed); + } else if (data_gen_type == DataGenType::ONES) { + std::fill(uint8_data.begin(), uint8_data.end(), 1); + } else if (data_gen_type == DataGenType::ONES_INT4) { + uint8_t packed_data = (9 << 4) | 9; + std::fill(uint8_data.begin(), uint8_data.end(), packed_data); + } else if (data_gen_type == DataGenType::ZEROS) { + std::fill(uint8_data.begin(), uint8_data.end(), 0); + } else { + std::fill(uint8_data.begin(), uint8_data.end(), 0); + } + break; + } + default: + // Default to float + float_data.resize(num_elements); + if (data_gen_type == DataGenType::RANDOM) { + generate_random_float_data(float_data, -1.0f, 1.0f, seed); + } else if (data_gen_type == DataGenType::RANDINT) { + generate_randint_float_data(float_data, -10, 10, seed); + } else if (data_gen_type == DataGenType::ONES) { + generate_ones_data(float_data); + } else if (data_gen_type == DataGenType::ZEROS) { + generate_zeros_data(float_data); + } else { + generate_zeros_data(float_data); + } + break; + } +} + +int64_t ValueSpec::numel() const { + if (spec_type == SpecType::Int || spec_type == SpecType::Float || + spec_type == SpecType::Bool) { + return 1; + } else if (spec_type == SpecType::IntList) { + return sizes.empty() ? 0 : sizes[0]; + } else { // Tensor + int64_t total = 1; + for (int64_t size : sizes) { + total *= size; + } + return total; + } +} + +size_t ValueSpec::nbytes() const { + size_t element_size = 0; + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + element_size = sizeof(float); + break; + case vkapi::kHalf: + element_size = sizeof(uint16_t); + break; + case vkapi::kInt: + element_size = sizeof(int32_t); + break; + case vkapi::kChar: + element_size = sizeof(int8_t); + break; + case vkapi::kByte: + element_size = sizeof(uint8_t); + break; + default: + element_size = sizeof(float); // Default fallback + break; + } + return numel() * element_size; +} + +std::string ValueSpec::to_string() const { + std::string result = "ValueSpec("; + + switch (spec_type) { + case SpecType::Tensor: + result += "type=Tensor, sizes=["; + break; + case SpecType::IntList: + result += "type=IntList, count="; + result += std::to_string(sizes.empty() ? 0 : sizes[0]); + result += ", data_gen="; + result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; + result += ")"; + return result; + case SpecType::Int: + result += "type=Int, value="; + result += std::to_string(get_int_value()); + result += ", data_gen="; + result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; + result += ")"; + return result; + case SpecType::Float: + result += "type=Float, value="; + result += std::to_string(get_float_value()); + result += ", data_gen="; + result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; + result += ")"; + return result; + case SpecType::Bool: + result += "type=Bool, value="; + result += get_bool_value() ? "true" : "false"; + result += ", data_gen="; + result += (data_gen_type == DataGenType::FIXED) ? "FIXED" : "RANDOM"; + result += ")"; + return result; + case SpecType::String: + result += "type=String, value=\""; + result += get_string_value(); + result += "\")"; + return result; + } + + for (size_t i = 0; i < sizes.size(); ++i) { + result += std::to_string(sizes[i]); + if (i < sizes.size() - 1) { + result += ", "; + } + } + result += "]"; + + if (spec_type == SpecType::Tensor) { + result += ", dtype="; + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + result += "float"; + break; + case vkapi::kHalf: + result += "half"; + break; + case vkapi::kInt: + result += "int32"; + break; + case vkapi::kChar: + result += "int8"; + break; + case vkapi::kByte: + result += "uint8"; + break; + default: + result += "unknown"; + break; + } + + result += ", memory_layout="; + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (memory_layout) { + case utils::kWidthPacked: + result += "WidthPacked"; + break; + case utils::kHeightPacked: + result += "HeightPacked"; + break; + case utils::kChannelsPacked: + result += "ChannelsPacked"; + break; + default: + result += "unknown"; + break; + } + + result += ", storage_type="; + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (storage_type) { + case utils::kTexture3D: + result += "Texture3D"; + break; + case utils::kBuffer: + result += "Buffer"; + break; + default: + result += "unknown"; + break; + } + } + + result += ", data_gen="; + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (data_gen_type) { + case DataGenType::FIXED: + result += "FIXED"; + break; + case DataGenType::RANDOM: + result += "RANDOM"; + break; + case DataGenType::RANDINT: + result += "RANDINT"; + break; + case DataGenType::RANDINT8: + result += "RANDINT8"; + break; + case DataGenType::RANDINT4: + result += "RANDINT4"; + break; + case DataGenType::ONES: + result += "ONES"; + break; + case DataGenType::ZEROS: + result += "ZEROS"; + break; + default: + result += "unknown"; + break; + } + result += ")"; + return result; +} + +// Additional ValueSpec methods +void ValueSpec::resize_data(size_t new_size) { + // Generate first so a deferred tensor keeps its data-gen pattern (resized, + // not pinned to zeros by the data_generated_ flag set below). + ensure_data_generated(); + ensure_unique_data(); + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + data_->float_data.resize(new_size); + break; + case vkapi::kHalf: + data_->half_data.resize(new_size); + break; + case vkapi::kInt: + data_->int32_data.resize(new_size); + break; + case vkapi::kChar: + data_->int8_data.resize(new_size); + break; + case vkapi::kByte: + data_->uint8_data.resize(new_size); + break; + default: + data_->float_data.resize(new_size); + break; + } + data_generated_ = true; +} + +void* ValueSpec::get_mutable_data_ptr() { + ensure_data_generated(); + ensure_unique_data(); + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + return data_->float_data.data(); + case vkapi::kHalf: + return data_->half_data.data(); + case vkapi::kInt: + return data_->int32_data.data(); + case vkapi::kChar: + return data_->int8_data.data(); + case vkapi::kByte: + return data_->uint8_data.data(); + default: + return data_->float_data.data(); + } +} + +float ValueSpec::get_element(size_t index) const { + ensure_data_generated(); + if (index >= static_cast(numel())) { + return 0.0f; + } + + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + return index < data_->float_data.size() ? data_->float_data[index] : 0.0f; + case vkapi::kHalf: + return index < data_->half_data.size() + ? half_to_float(data_->half_data[index]) + : 0.0f; + case vkapi::kInt: + return index < data_->int32_data.size() + ? static_cast(data_->int32_data[index]) + : 0.0f; + case vkapi::kChar: + return index < data_->int8_data.size() + ? static_cast(data_->int8_data[index]) + : 0.0f; + case vkapi::kByte: + return index < data_->uint8_data.size() + ? static_cast(data_->uint8_data[index]) + : 0.0f; + default: + return 0.0f; + } +} + +const void* ValueSpec::get_data_ptr() const { + ensure_data_generated(); + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (dtype) { + case vkapi::kFloat: + return data_->float_data.data(); + case vkapi::kHalf: + return data_->half_data.data(); + case vkapi::kInt: + return data_->int32_data.data(); + case vkapi::kChar: + return data_->int8_data.data(); + case vkapi::kByte: + return data_->uint8_data.data(); + default: + throw std::runtime_error("Unsupported data type for get_data_ptr"); + } +} + +void generate_random_float_data( + std::vector& data, + float min_val, + float max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_real_distribution dis(min_val, max_val); + for (auto& val : data) { + val = dis(gen); + } +} + +void generate_random_int_data( + std::vector& data, + int min_val, + int max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(min_val, max_val); + for (auto& val : data) { + val = dis(gen); + } +} + +void generate_randint_float_data( + std::vector& data, + int min_val, + int max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(min_val, max_val); + for (auto& val : data) { + val = static_cast(dis(gen)); + } +} + +void generate_randint_half_data( + std::vector& data, + int min_val, + int max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(min_val, max_val); + for (auto& val : data) { + val = float_to_half(static_cast(dis(gen))); + } +} + +void generate_ones_data(std::vector& data) { + std::fill(data.begin(), data.end(), 1.0f); +} + +void generate_random_int8_data( + std::vector& data, + int8_t min_val, + int8_t max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(min_val, max_val); + for (auto& val : data) { + val = static_cast(dis(gen)); + } +} + +void generate_random_uint8_data( + std::vector& data, + uint8_t min_val, + uint8_t max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(min_val, max_val); + for (auto& val : data) { + val = static_cast(dis(gen)); + } +} + +void generate_random_int4_data( + std::vector& data, + int8_t min_val, + int8_t max_val, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(min_val, max_val); + for (auto& val : data) { + val = static_cast(dis(gen)); + } +} + +void generate_random_2xint4_data(std::vector& data, int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(-8, 7); // Signed 4-bit range + for (auto& val : data) { + // Generate two separate 4-bit values + int8_t lower_4bits = static_cast(dis(gen)) & 0x0F; + int8_t upper_4bits = static_cast(dis(gen)) & 0x0F; + // Pack them into a single 8-bit value + val = (upper_4bits << 4) | lower_4bits; + } +} + +void generate_random_2xint4_data( + std::vector& data, + int explicit_seed) { + std::mt19937 gen(get_seed_or_explicit(explicit_seed)); + std::uniform_int_distribution dis(0, 15); // Unsigned 4-bit range + for (auto& val : data) { + // Generate two separate 4-bit values + uint8_t lower_4bits = static_cast(dis(gen)) & 0x0F; + uint8_t upper_4bits = static_cast(dis(gen)) & 0x0F; + // Pack them into a single 8-bit value + val = (upper_4bits << 4) | lower_4bits; + } +} + +void generate_zeros_data(std::vector& data) { + std::fill(data.begin(), data.end(), 0.0f); +} + +// Correctness checking against reference data +bool ValueSpec::validate_against_reference( + float abs_tolerance, + float rel_tolerance) const { + // Only validate float and half tensors. For half tensors, convert the + // computed half data to float for comparison against the fp32 reference. + if (!is_tensor() || (dtype != vkapi::kFloat && dtype != vkapi::kHalf)) { + return true; // Skip validation for non-float/half or non-tensor types + } + + // For kHalf, materialize the GPU output as float so the same tolerance + // machinery can compare against the (always-float) reference data. + std::vector half_as_float; + if (dtype == vkapi::kHalf) { + const auto& half_bits = get_half_data(); + half_as_float.resize(half_bits.size()); + for (size_t i = 0; i < half_bits.size(); ++i) { + half_as_float[i] = half_to_float(half_bits[i]); + } + } + // Materialize computed data as float32 for comparison. The dtype is + // guaranteed to be float or half by the early-out above. + const std::vector& computed_data = + (dtype == vkapi::kHalf) ? half_as_float : get_float_data(); + const auto& reference_data = get_ref_float_data(); + + // Skip validation if no reference data is available + if (reference_data.empty()) { + return true; + } + + // Check if sizes match + if (computed_data.size() != reference_data.size()) { + if (debugging()) { + std::cout << "Size mismatch: computed=" << computed_data.size() + << ", reference=" << reference_data.size() << std::endl; + } + return false; + } + + // Element-wise comparison with both absolute and relative tolerance + size_t num_mismatched = 0; + size_t first_mismatch = 0; + for (size_t i = 0; i < computed_data.size(); ++i) { + float diff = std::abs(computed_data[i] - reference_data[i]); + float abs_ref = std::abs(reference_data[i]); + + // Check if either absolute or relative tolerance condition is satisfied + bool abs_tolerance_ok = diff <= abs_tolerance; + bool rel_tolerance_ok = diff <= rel_tolerance * abs_ref; + + if (!abs_tolerance_ok && !rel_tolerance_ok) { + if (num_mismatched == 0) { + first_mismatch = i; + std::cout << "Mismatch at element " << i + << ": computed=" << computed_data[i] + << ", reference=" << reference_data[i] << ", diff=" << diff + << ", abs_tolerance=" << abs_tolerance + << ", rel_tolerance=" << rel_tolerance + << ", rel_threshold=" << (rel_tolerance * abs_ref) + << std::endl; + } + num_mismatched++; + } + } + if (num_mismatched > 0) { + std::cout << " total mismatched: " << num_mismatched << " / " + << computed_data.size() << " (first at " << first_mismatch << ")" + << std::endl; + // For 2D outputs, print a per-16x16-tile mismatch-count map to expose + // the spatial structure of the failure (e.g. zeroed MMA subtiles). + if (sizes.size() == 2) { + const int64_t Mr = sizes[0]; + const int64_t Nc = sizes[1]; + std::cout << " 16x16-tile mismatch counts (rows=M/16, cols=N/16):" + << std::endl; + for (int64_t ti = 0; ti < (Mr + 15) / 16; ++ti) { + std::cout << " "; + for (int64_t tj = 0; tj < (Nc + 15) / 16; ++tj) { + int count = 0; + for (int64_t r = ti * 16; r < std::min(Mr, (ti + 1) * 16); ++r) { + for (int64_t c = tj * 16; c < std::min(Nc, (tj + 1) * 16); ++c) { + float diff = std::abs( + computed_data[r * Nc + c] - reference_data[r * Nc + c]); + float abs_ref = std::abs(reference_data[r * Nc + c]); + if (diff > abs_tolerance && diff > rel_tolerance * abs_ref) { + count++; + } + } + } + std::cout << std::setw(4) << count; + } + std::cout << std::endl; + } + } + return false; + } + + if (debugging()) { + std::cout << "Correctness validation PASSED" << std::endl; + } + return true; +} + +// Ensure data is generated for this ValueSpec +void ValueSpec::ensure_data_generated(int seed) const { + if (data_generated_) { + return; + } + generate_tensor_data(seed); + data_generated_ = true; +} + +void ValueSpec::share_data_from(const ValueSpec& other) { + if (!is_tensor() || !other.is_tensor()) { + return; + } + // Materialize the source first: sharing an ungenerated payload would let a + // later access materialize each spec independently under different seeds. + other.ensure_data_generated(); + data_ = other.data_; + data_generated_ = true; +} + +void ValueSpec::share_reference_from(const ValueSpec& other) { + if (!is_tensor() || !other.is_tensor()) { + return; + } + reference_data_ = other.reference_data_; +} + +// ValueSpec data printing utilities +void print_valuespec_data( + const ValueSpec& spec, + const std::string& name, + const bool print_ref_data, + size_t max_elements, + int precision) { + std::cout << "\n" << name << " Data:" << std::endl; + std::cout << " Type: " << spec.to_string() << std::endl; + + if (!spec.is_tensor()) { + if (spec.is_int()) { + std::cout << " Value: " << spec.get_int_value() << std::endl; + } else if (spec.is_int_list()) { + const auto& int_list = spec.get_int_list(); + std::cout << " Values: ["; + size_t print_count = std::min(max_elements, int_list.size()); + for (size_t i = 0; i < print_count; ++i) { + std::cout << int_list[i]; + if (i < print_count - 1) { + std::cout << ", "; + } + } + if (int_list.size() > max_elements) { + std::cout << ", ... (" << (int_list.size() - max_elements) << " more)"; + } + std::cout << "]" << std::endl; + } + return; + } + + // Print tensor data + size_t total_elements = spec.numel(); + size_t print_count = std::min(max_elements, total_elements); + + std::cout << " Total elements: " << total_elements << std::endl; + std::cout << " Data (first " << print_count << " elements): ["; + + std::cout << std::fixed << std::setprecision(precision); + + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (spec.dtype) { + case vkapi::kFloat: { + auto data = spec.get_float_data().data(); + if (print_ref_data) { + data = spec.get_ref_float_data().data(); + } + for (size_t i = 0; i < print_count; ++i) { + std::cout << data[i]; + if (i < print_count - 1) { + std::cout << ", "; + } + } + break; + } + case vkapi::kHalf: { + if (print_ref_data) { + const auto& ref = spec.get_ref_float_data(); + for (size_t i = 0; i < print_count; ++i) { + std::cout << ref[i]; + if (i < print_count - 1) { + std::cout << ", "; + } + } + break; + } + const auto& data = spec.get_half_data(); + for (size_t i = 0; i < print_count; ++i) { + // Convert IEEE 754 half-precision bit pattern back to float. + float value = half_to_float(data[i]); + std::cout << value; + if (i < print_count - 1) { + std::cout << ", "; + } + } + break; + } + case vkapi::kInt: { + const auto& data = spec.get_int32_data(); + for (size_t i = 0; i < print_count; ++i) { + std::cout << data[i]; + if (i < print_count - 1) { + std::cout << ", "; + } + } + break; + } + case vkapi::kChar: { + const auto& data = spec.get_int8_data(); + if (spec.is_int4()) { + // Print each 4-bit value individually + size_t element_count = 0; + for (size_t i = 0; i < data.size() && element_count < print_count; + ++i) { + // Extract lower 4 bits (signed) + int8_t lower_4bits = data[i] & 0x0F; + if (lower_4bits > 7) { + lower_4bits -= 16; // Convert to signed + } + std::cout << static_cast(lower_4bits); + element_count++; + + if (element_count < print_count) { + std::cout << ", "; + // Extract upper 4 bits (signed) + int8_t upper_4bits = (data[i] >> 4) & 0x0F; + if (upper_4bits > 7) { + upper_4bits -= 16; // Convert to signed + } + std::cout << static_cast(upper_4bits); + element_count++; + + if (element_count < print_count) { + std::cout << ", "; + } + } + } + } else { + for (size_t i = 0; i < print_count; ++i) { + std::cout << static_cast(data[i]); + if (i < print_count - 1) { + std::cout << ", "; + } + } + } + break; + } + case vkapi::kByte: { + const auto& data = spec.get_uint8_data(); + if (spec.is_int4()) { + // Print each 4-bit value individually + size_t element_count = 0; + for (size_t i = 0; i < data.size() && element_count < print_count; + ++i) { + // Extract lower 4 bits + uint8_t lower_4bits = data[i] & 0x0F; + std::cout << static_cast(lower_4bits); + element_count++; + + if (element_count < print_count) { + std::cout << ", "; + // Extract upper 4 bits + uint8_t upper_4bits = (data[i] >> 4) & 0x0F; + std::cout << static_cast(upper_4bits); + element_count++; + + if (element_count < print_count) { + std::cout << ", "; + } + } + } + } else { + for (size_t i = 0; i < print_count; ++i) { + std::cout << static_cast(data[i]); + if (i < print_count - 1) { + std::cout << ", "; + } + } + } + break; + } + default: + std::cout << "unsupported data type"; + break; + } + + if (total_elements > max_elements) { + std::cout << ", ... (" << (total_elements - max_elements) << " more)"; + } + std::cout << "]" << std::endl; + + // Print some statistics for tensor data + if (total_elements > 0) { + float min_val = 0.0f, max_val = 0.0f, sum = 0.0f; + bool first = true; + + for (size_t i = 0; i < total_elements; ++i) { + float val = spec.get_element(i); + if (first) { + min_val = max_val = val; + first = false; + } else { + min_val = std::min(min_val, val); + max_val = std::max(max_val, val); + } + sum += val; + } + + float mean = sum / total_elements; + std::cout << " Statistics: min=" << std::setprecision(precision) << min_val + << ", max=" << max_val << ", mean=" << mean << ", sum=" << sum + << std::endl; + } +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/value_spec.h b/backends/vulkan/test/ops/benchmark/framework/value_spec.h new file mode 100644 index 00000000000..ebb87e2c323 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/value_spec.h @@ -0,0 +1,404 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +using namespace vkcompute; + +// +// ValueSpec class +// + +enum class SpecType { Tensor, IntList, Int, Float, Bool, String }; + +// Data generation types +enum class DataGenType { + FIXED, + RANDOM, + RANDOM_SCALES, + RANDINT, + RANDINT8, + RANDINT4, + ONES, + ONES_INT4, + ZEROS +}; + +// Value specification struct +struct ValueSpec { + std::vector sizes; + vkapi::ScalarType dtype; + utils::GPUMemoryLayout memory_layout; + utils::StorageType storage_type; + SpecType spec_type; + DataGenType data_gen_type; + bool is_constant_tensor; + bool is_none_flag; + bool is_int4_tensor; + std::string string_data; + + ValueSpec( + const std::vector& sizes, + vkapi::ScalarType dtype, + utils::StorageType storage_type = utils::kTexture3D, + utils::GPUMemoryLayout memory_layout = utils::kWidthPacked) + : sizes(sizes), + dtype(dtype), + memory_layout(memory_layout), + storage_type(storage_type), + spec_type(SpecType::Tensor), + data_gen_type(DataGenType::ZEROS), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(false) { + // Data generation is deferred until first access (any data getter or + // ensure_data_generated() triggers it). + } + + // Constructor for tensor with custom data generation type + ValueSpec( + const std::vector& sizes, + vkapi::ScalarType dtype, + utils::StorageType storage_type, + utils::GPUMemoryLayout memory_layout, + DataGenType data_gen_type) + : sizes(sizes), + dtype(dtype), + memory_layout(memory_layout), + storage_type(storage_type), + spec_type(SpecType::Tensor), + data_gen_type(data_gen_type), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(false) { + // Data generation is deferred until first access (any data getter or + // ensure_data_generated() triggers it). + } + + // Constructor for single int + ValueSpec(int32_t value) + : sizes({1}), + dtype(vkapi::kInt), + memory_layout(utils::kWidthPacked), + storage_type(utils::kTexture3D), + spec_type(SpecType::Int), + data_gen_type(DataGenType::FIXED), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(true) { + data_->int32_data.push_back(value); + } + + // Constructor for single float + ValueSpec(float value) + : sizes({1}), + dtype(vkapi::kFloat), + memory_layout(utils::kWidthPacked), + storage_type(utils::kTexture3D), + spec_type(SpecType::Float), + data_gen_type(DataGenType::FIXED), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(true) { + data_->float_data.push_back(value); + } + + // Constructor for single bool + ValueSpec(bool value) + : sizes({1}), + dtype(vkapi::kInt), + memory_layout(utils::kWidthPacked), + storage_type(utils::kTexture3D), + spec_type(SpecType::Bool), + data_gen_type(DataGenType::FIXED), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(true) { + data_->int32_data.push_back(value ? 1 : 0); + } + + // Constructor for int list + ValueSpec(const std::vector& values) + : sizes({static_cast(values.size())}), + dtype(vkapi::kInt), + memory_layout(utils::kWidthPacked), + storage_type(utils::kTexture3D), + spec_type(SpecType::IntList), + data_gen_type(DataGenType::FIXED), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(true) { + data_->int32_data = values; + } + + // Factory method for string (avoids ambiguity with vector constructor) + static ValueSpec make_string(const std::string& value) { + ValueSpec spec; + spec.sizes = {1}; + spec.dtype = vkapi::kInt; + spec.memory_layout = utils::kWidthPacked; + spec.storage_type = utils::kTexture3D; + spec.spec_type = SpecType::String; + spec.data_gen_type = DataGenType::FIXED; + spec.is_constant_tensor = false; + spec.is_none_flag = false; + spec.is_int4_tensor = false; + spec.data_generated_ = true; + spec.string_data = value; + return spec; + } + + // Default constructor + ValueSpec() + : dtype(vkapi::kFloat), + memory_layout(utils::kWidthPacked), + storage_type(utils::kTexture3D), + spec_type(SpecType::Tensor), + data_gen_type(DataGenType::ZEROS), + is_constant_tensor(false), + is_none_flag(false), + is_int4_tensor(false), + data_generated_(false) {} + + int64_t numel() const; + size_t nbytes() const; + std::string to_string() const; + + bool is_tensor() const { + return spec_type == SpecType::Tensor; + } + bool is_int_list() const { + return spec_type == SpecType::IntList; + } + bool is_int() const { + return spec_type == SpecType::Int; + } + bool is_float() const { + return spec_type == SpecType::Float; + } + bool is_bool() const { + return spec_type == SpecType::Bool; + } + bool is_string() const { + return spec_type == SpecType::String; + } + + int32_t get_int_value() const { + ensure_data_generated(); + return data_->int32_data.empty() ? 0 : data_->int32_data[0]; + } + float get_float_value() const { + ensure_data_generated(); + return data_->float_data.empty() ? 0.0f : data_->float_data[0]; + } + bool get_bool_value() const { + ensure_data_generated(); + return data_->int32_data.empty() ? false : (data_->int32_data[0] != 0); + } + const std::string& get_string_value() const { + return string_data; + } + const std::vector& get_int_list() const { + ensure_data_generated(); + return data_->int32_data; + } + const std::vector& get_tensor_sizes() const { + return sizes; + } + + // References and pointers into tensor data must not be held across any other + // access to the same spec: a mutable access may detach the shared payload, + // leaving a previously returned reference bound to the old payload. Consume + // immediately. + const std::vector& get_float_data() const { + ensure_data_generated(); + return data_->float_data; + } + const std::vector& get_int32_data() const { + ensure_data_generated(); + return data_->int32_data; + } + const std::vector& get_half_data() const { + ensure_data_generated(); + return data_->half_data; + } + const std::vector& get_int8_data() const { + ensure_data_generated(); + return data_->int8_data; + } + const std::vector& get_uint8_data() const { + ensure_data_generated(); + return data_->uint8_data; + } + + std::vector& get_float_data() { + ensure_data_generated(); + ensure_unique_data(); + return data_->float_data; + } + std::vector& get_int32_data() { + ensure_data_generated(); + ensure_unique_data(); + return data_->int32_data; + } + std::vector& get_half_data() { + ensure_data_generated(); + ensure_unique_data(); + return data_->half_data; + } + std::vector& get_int8_data() { + ensure_data_generated(); + ensure_unique_data(); + return data_->int8_data; + } + std::vector& get_uint8_data() { + ensure_data_generated(); + ensure_unique_data(); + return data_->uint8_data; + } + + const std::vector& get_ref_float_data() const { + return reference_data_->float_data; + } + const std::vector& get_ref_int32_data() const { + return reference_data_->int32_data; + } + const std::vector& get_ref_half_data() const { + return reference_data_->half_data; + } + const std::vector& get_ref_int8_data() const { + return reference_data_->int8_data; + } + const std::vector& get_ref_uint8_data() const { + return reference_data_->uint8_data; + } + + std::vector& get_ref_float_data() { + ensure_unique_reference_data(); + return reference_data_->float_data; + } + std::vector& get_ref_int32_data() { + ensure_unique_reference_data(); + return reference_data_->int32_data; + } + std::vector& get_ref_half_data() { + ensure_unique_reference_data(); + return reference_data_->half_data; + } + std::vector& get_ref_int8_data() { + ensure_unique_reference_data(); + return reference_data_->int8_data; + } + std::vector& get_ref_uint8_data() { + ensure_unique_reference_data(); + return reference_data_->uint8_data; + } + + void resize_data(size_t new_size); + void* get_mutable_data_ptr(); + float get_element(size_t index) const; + + // Data generation methods for deferred generation and caching. + // + // ValueSpec is not thread-safe: lazy materialization and copy-on-write + // detach mutate shared state from const methods. Test cases are built and + // executed on a single thread. + // + // Implicit materialization (any data getter, resize_data) consumes the + // global seed counter. Callers needing deterministic data must call + // ensure_data_generated(explicit_seed) before any other access; a later + // seeded call is a no-op once data is generated. + bool is_data_generated() const { + return data_generated_; + } + void ensure_data_generated(int seed = -1) const; + void share_data_from(const ValueSpec& other); + void share_reference_from(const ValueSpec& other); + + // Set/get constant flag + bool is_constant() const { + return is_constant_tensor; + } + void set_constant(bool is_constant) { + is_constant_tensor = is_constant; + } + + // Set/get none flag + bool is_none() const { + return is_none_flag; + } + + void set_none(bool is_none) { + is_none_flag = is_none; + } + + // Set/get int4 flag + bool is_int4() const { + return is_int4_tensor; + } + void set_int4(bool is_int4) { + is_int4_tensor = is_int4; + } + + const void* get_data_ptr() const; + + // Correctness checking against reference data + // Returns true if computed data matches reference data within tolerance + // Only validates float tensors as specified in requirements + bool validate_against_reference( + float abs_tolerance = 2e-3f, + float rel_tolerance = 1e-3f) const; + + private: + struct TensorData { + std::vector float_data; + std::vector int32_data; + std::vector half_data; + std::vector int8_data; + std::vector uint8_data; + }; + + void ensure_unique_data() const; + void ensure_unique_reference_data() const; + void generate_tensor_data(int seed = -1) const; + + mutable bool data_generated_ = false; + mutable std::shared_ptr data_ = std::make_shared(); + mutable std::shared_ptr reference_data_ = + std::make_shared(); +}; + +void print_valuespec_data( + const ValueSpec& spec, + const std::string& name = "ValueSpec", + const bool print_ref_data = false, + size_t max_elements = 20, + int precision = 6); + +// Half-precision conversion utilities +uint16_t float_to_half(float value); +float half_to_float(uint16_t half_val); + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/weight_utils.cpp b/backends/vulkan/test/ops/benchmark/framework/weight_utils.cpp new file mode 100644 index 00000000000..5161178d978 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/weight_utils.cpp @@ -0,0 +1,247 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include "weight_utils.h" +#include "config.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +ValueRef quantized_weights_canvas( + ComputeGraph& graph, + const ValueRef weight_ref) { + const auto original_sizes = graph.sizes_of(weight_ref); + + std::vector sorted_sizes = original_sizes; + std::sort(sorted_sizes.begin(), sorted_sizes.end(), std::greater()); + int64_t largest1 = sorted_sizes.size() > 0 ? sorted_sizes[0] : 0; + + std::vector final_sizes = {1, largest1, largest1}; + + // Debug logging if debugging flag is set + if (debugging()) { + std::cout << "Debug: Creating quantized weights canvas tensor" << std::endl; + std::cout << "Debug: Original sizes: ["; + for (size_t i = 0; i < original_sizes.size(); ++i) { + std::cout << original_sizes[i]; + if (i < original_sizes.size() - 1) + std::cout << ", "; + } + std::cout << "]" << std::endl; + std::cout << "Debug: Canvas sizes: ["; + for (size_t i = 0; i < final_sizes.size(); ++i) { + std::cout << final_sizes[i]; + if (i < final_sizes.size() - 1) + std::cout << ", "; + } + std::cout << "]" << std::endl; + } + + ValueRef packed_weight = graph.add_tensor( + final_sizes, vkapi::kInt, utils::kTexture3D, utils::kWidthPacked); + + std::string kernel_name = "packed_int32_canvas"; + add_storage_type_suffix(kernel_name, graph.storage_type_of(packed_weight)); + + graph.prepack_nodes().emplace_back(new PrepackNode( + graph, + VK_KERNEL_FROM_STR(kernel_name), + graph.create_gwg(packed_weight), + graph.create_lwg(packed_weight), + weight_ref, + packed_weight, + // UBOs + {graph.logical_limits_ubo(packed_weight)}, + // Specialization constants + {}, + // Push Constants + {})); + + return packed_weight; +} + +ValueRef float_tensor_canvas(ComputeGraph& graph, const ValueRef weight_ref) { + const auto original_sizes = graph.sizes_of(weight_ref); + + std::vector sorted_sizes = original_sizes; + std::sort(sorted_sizes.begin(), sorted_sizes.end(), std::greater()); + int64_t largest1 = sorted_sizes.size() > 0 ? sorted_sizes[0] : 0; + + std::vector final_sizes = {1, largest1, largest1}; + + // Debug logging if debugging flag is set + if (debugging()) { + std::cout << "Debug: Creating float tensor canvas" << std::endl; + std::cout << "Debug: Original sizes: ["; + for (size_t i = 0; i < original_sizes.size(); ++i) { + std::cout << original_sizes[i]; + if (i < original_sizes.size() - 1) + std::cout << ", "; + } + std::cout << "]" << std::endl; + std::cout << "Debug: Canvas sizes: ["; + for (size_t i = 0; i < final_sizes.size(); ++i) { + std::cout << final_sizes[i]; + if (i < final_sizes.size() - 1) + std::cout << ", "; + } + std::cout << "]" << std::endl; + } + + ValueRef packed_weight = graph.add_tensor( + final_sizes, vkapi::kFloat, utils::kTexture3D, utils::kWidthPacked); + + graph.prepack_nodes().emplace_back(new PrepackNode( + graph, + VK_KERNEL_FROM_STR("float_canvas"), + graph.create_gwg(packed_weight), + graph.create_lwg(packed_weight), + weight_ref, + packed_weight, + // UBOs + {graph.logical_limits_ubo(packed_weight)}, + // Specialization constants + {}, + // Push Constants + {})); + + return packed_weight; +} + +// Compute weight sums for quantized operations (linear and convolution) +void compute_weight_sums( + ValueSpec& weight_sums, + const ValueSpec& quantized_weight, + int64_t out_features, + int64_t elements_per_output_feature) { + auto& weight_sums_data = weight_sums.get_int32_data(); + auto& quantized_weight_data = quantized_weight.get_int8_data(); + + // Don't resize down - the buffer may be pre-allocated with aligned size. + // Only resize up if needed. + if (weight_sums_data.size() < static_cast(out_features)) { + weight_sums_data.resize(out_features); + } + + // For each output feature, compute the sum of quantized weights + for (int64_t out_f = 0; out_f < out_features; ++out_f) { + int32_t sum = 0; + for (int64_t elem = 0; elem < elements_per_output_feature; ++elem) { + // Weight indexing depends on the layout: + // For linear: [out_features, in_features] -> out_f * + // elements_per_output_feature + elem For conv2d: [C_out, C_in * K_h * + // K_w] -> out_f * elements_per_output_feature + elem + int64_t weight_idx = out_f * elements_per_output_feature + elem; + sum += static_cast(quantized_weight_data[weight_idx]); + } + weight_sums_data[out_f] = sum; + } +} + +// Compute weight sums for 4D quantized conv2d operations +// Weight layout: [C_out, K_h, K_w, align_up_4(C_in_per_group)] +void compute_weight_sums_4d( + ValueSpec& weight_sums, + const ValueSpec& quantized_weight, + int64_t out_channels, + int64_t kernel_h, + int64_t kernel_w, + int64_t aligned_in_channels) { + auto& weight_sums_data = weight_sums.get_int32_data(); + auto& quantized_weight_data = quantized_weight.get_int8_data(); + + weight_sums_data.resize(out_channels); + + // For each output channel, compute the sum of quantized weights + for (int64_t out_c = 0; out_c < out_channels; ++out_c) { + int32_t sum = 0; + + for (int64_t kh = 0; kh < kernel_h; ++kh) { + for (int64_t kw = 0; kw < kernel_w; ++kw) { + for (int64_t in_c = 0; in_c < aligned_in_channels; ++in_c) { + // Weight indexing: [out_c, kh, kw, in_c] + int64_t weight_idx = + out_c * (kernel_h * kernel_w * aligned_in_channels) + + kh * (kernel_w * aligned_in_channels) + kw * aligned_in_channels + + in_c; + sum += static_cast(quantized_weight_data[weight_idx]); + } + } + } + + weight_sums_data[out_c] = sum; + } +} + +// Helper function to unpack 4-bit values from uint8 (same as in +// q4gsw_linear.cpp) +std::pair unpack_4bit_utils(uint8_t packed) { + // Extract lower 4 bits and upper 4 bits + int8_t lower = packed & 0x0F; + int8_t upper = (packed >> 4) & 0x0F; + + // Subtract 8 from unpacked 4-bit values + lower -= 8; + upper -= 8; + + return std::make_pair(lower, upper); +} + +// Compute weight sums for 4-bit group symmetric quantized weights +void compute_weight_sums_4bit_grouped( + ValueSpec& weight_sums, + const ValueSpec& quantized_weight, + int64_t num_groups, + int64_t out_features, + int64_t group_size) { + auto& weight_sums_data = weight_sums.get_int32_data(); + auto& quantized_weight_data = quantized_weight.get_uint8_data(); + + // Resize to [num_groups, out_features] + weight_sums_data.resize(num_groups * out_features); + + // For each group and each output feature, compute the sum of quantized + // weights in that group + for (int64_t group_idx = 0; group_idx < num_groups; ++group_idx) { + for (int64_t out_f = 0; out_f < out_features; ++out_f) { + int32_t sum = 0; + + // Sum weights for this group and output feature + for (int64_t in_group = 0; in_group < group_size; ++in_group) { + int64_t in_f = group_idx * group_size + in_group; + + // Get packed weight value - weight matrix is [N, K/2] + int64_t weight_idx = + out_f * ((num_groups * group_size) / 2) + (in_f / 2); + uint8_t packed_weight = quantized_weight_data[weight_idx]; + + // Unpack 4-bit weight + auto unpacked = unpack_4bit_utils(packed_weight); + int8_t weight_4bit = (in_f % 2 == 0) ? unpacked.first : unpacked.second; + + sum += static_cast(weight_4bit); + } + + // Store sum for this group and output feature + int64_t sums_idx = group_idx * out_features + out_f; + weight_sums_data[sums_idx] = sum; + } + } +} + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/framework/weight_utils.h b/backends/vulkan/test/ops/benchmark/framework/weight_utils.h new file mode 100644 index 00000000000..7c9319703b9 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/framework/weight_utils.h @@ -0,0 +1,52 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#include "value_spec.h" + +#include + +namespace executorch { +namespace vulkan { +namespace prototyping { + +ValueRef quantized_weights_canvas( + ComputeGraph& graph, + const ValueRef weight_ref); + +ValueRef float_tensor_canvas(ComputeGraph& graph, const ValueRef weight_ref); + +// Compute weight sums for quantized operations (linear and convolution) +void compute_weight_sums( + ValueSpec& weight_sums, + const ValueSpec& quantized_weight, + int64_t out_features, + int64_t elements_per_output_feature); + +// Compute weight sums for 4D quantized conv2d operations +// Weight layout: [C_out, K_h, K_w, align_up_4(C_in_per_group)] +void compute_weight_sums_4d( + ValueSpec& weight_sums, + const ValueSpec& quantized_weight, + int64_t out_channels, + int64_t kernel_h, + int64_t kernel_w, + int64_t aligned_in_channels); + +// Compute weight sums for 4-bit group symmetric quantized weights +void compute_weight_sums_4bit_grouped( + ValueSpec& weight_sums, + const ValueSpec& quantized_weight, + int64_t num_groups, + int64_t out_features, + int64_t group_size); + +} // namespace prototyping +} // namespace vulkan +} // namespace executorch diff --git a/backends/vulkan/test/ops/benchmark/targets.bzl b/backends/vulkan/test/ops/benchmark/targets.bzl new file mode 100644 index 00000000000..d53f252cc65 --- /dev/null +++ b/backends/vulkan/test/ops/benchmark/targets.bzl @@ -0,0 +1,109 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") +load( + "@fbsource//xplat/executorch/backends/vulkan:targets.bzl", + "get_platforms", + "vulkan_spv_shader_lib", +) + +def define_common_targets(is_fbcode = False): + if is_fbcode: + return + + # Shader library from GLSL files + runtime.filegroup( + name = "custom_ops_shaders", + srcs = native.glob([ + "test_ops/glsl/**/*", + ]), + visibility = ["PUBLIC"], + ) + + vulkan_spv_shader_lib( + name = "custom_ops_shaderlib", + spv_filegroups = { + ":custom_ops_shaders": "test_ops/glsl", + }, + is_fbcode = is_fbcode, + ) + + # Prototyping utilities library + runtime.cxx_library( + name = "prototyping_utils", + srcs = [ + "framework/cm_utils.cpp", + "framework/config.cpp", + "framework/conv2d_utils.cpp", + "framework/device_info.cpp", + "framework/registry.cpp", + "framework/results.cpp", + "framework/runner.cpp", + "framework/test_case.cpp", + "framework/value_spec.cpp", + "framework/weight_utils.cpp", + ], + headers = [ + "framework/cm_utils.h", + "framework/config.h", + "framework/conv2d_utils.h", + "framework/device_info.h", + "framework/labels.h", + "framework/registry.h", + "framework/results.h", + "framework/runner.h", + "framework/test_case.h", + "framework/utils.h", + "framework/value_spec.h", + "framework/weight_utils.h", + ], + exported_headers = [ + "framework/cm_utils.h", + "framework/config.h", + "framework/conv2d_utils.h", + "framework/device_info.h", + "framework/labels.h", + "framework/registry.h", + "framework/results.h", + "framework/runner.h", + "framework/test_case.h", + "framework/utils.h", + "framework/value_spec.h", + "framework/weight_utils.h", + ], + platforms = get_platforms(), + deps = [ + "//executorch/backends/vulkan:vulkan_graph_runtime", + ], + visibility = ["PUBLIC"], + ) + + # Operator implementations library + runtime.cxx_library( + name = "operator_implementations", + srcs = native.glob([ + "test_ops/impl/*.cpp", + ]), + platforms = get_platforms(), + deps = [ + "//executorch/backends/vulkan:vulkan_graph_runtime", + ":custom_ops_shaderlib", + ], + visibility = ["PUBLIC"], + link_whole = True, + ) + + runtime.cxx_binary( + name = "etvk_bench", + srcs = ["etvk_bench.cpp"] + native.glob(["cases/**/*.cpp"]), + headers = native.glob(["cases/**/*.h"]), + platforms = get_platforms(), + define_static_target = False, + deps = [ + ":custom_ops_shaderlib", + ":operator_implementations", + ":prototyping_utils", + "//executorch/backends/vulkan:vulkan_graph_runtime", + ], + external_deps = [ + "gflags", + ], + ) diff --git a/backends/vulkan/test/custom_ops/glsl/add.yaml b/backends/vulkan/test/ops/benchmark/test_ops/glsl/add.yaml similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/add.yaml rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/add.yaml diff --git a/backends/vulkan/test/custom_ops/glsl/add_buffer.glsl b/backends/vulkan/test/ops/benchmark/test_ops/glsl/add_buffer.glsl similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/add_buffer.glsl rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/add_buffer.glsl diff --git a/backends/vulkan/test/custom_ops/glsl/add_texture.glsl b/backends/vulkan/test/ops/benchmark/test_ops/glsl/add_texture.glsl similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/add_texture.glsl rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/add_texture.glsl diff --git a/backends/vulkan/test/custom_ops/glsl/float_canvas.glsl b/backends/vulkan/test/ops/benchmark/test_ops/glsl/float_canvas.glsl similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/float_canvas.glsl rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/float_canvas.glsl diff --git a/backends/vulkan/test/custom_ops/glsl/packed_int32_canvas_buffer.glsl b/backends/vulkan/test/ops/benchmark/test_ops/glsl/packed_int32_canvas_buffer.glsl similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/packed_int32_canvas_buffer.glsl rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/packed_int32_canvas_buffer.glsl diff --git a/backends/vulkan/test/custom_ops/glsl/packed_int32_canvas_texture3d.glsl b/backends/vulkan/test/ops/benchmark/test_ops/glsl/packed_int32_canvas_texture3d.glsl similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/packed_int32_canvas_texture3d.glsl rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/packed_int32_canvas_texture3d.glsl diff --git a/backends/vulkan/test/custom_ops/glsl/q4gsw_linear_gemv__w_4x8.glsl b/backends/vulkan/test/ops/benchmark/test_ops/glsl/q4gsw_linear_gemv__w_4x8.glsl similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/q4gsw_linear_gemv__w_4x8.glsl rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/q4gsw_linear_gemv__w_4x8.glsl diff --git a/backends/vulkan/test/custom_ops/glsl/q4gsw_linear_gemv__w_4x8.yaml b/backends/vulkan/test/ops/benchmark/test_ops/glsl/q4gsw_linear_gemv__w_4x8.yaml similarity index 100% rename from backends/vulkan/test/custom_ops/glsl/q4gsw_linear_gemv__w_4x8.yaml rename to backends/vulkan/test/ops/benchmark/test_ops/glsl/q4gsw_linear_gemv__w_4x8.yaml diff --git a/backends/vulkan/test/custom_ops/impl/AddPrototype.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/AddPrototype.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/AddPrototype.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/AddPrototype.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestConv1dDW.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv1dDW.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestConv1dDW.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv1dDW.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestConv1dPW.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv1dPW.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestConv1dPW.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv1dPW.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestConv2d.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2d.cpp similarity index 88% rename from backends/vulkan/test/custom_ops/impl/TestConv2d.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2d.cpp index 343f416c6a1..bad30507ac1 100644 --- a/backends/vulkan/test/custom_ops/impl/TestConv2d.cpp +++ b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2d.cpp @@ -9,8 +9,8 @@ #include #include -#include -#include +#include +#include #include @@ -32,8 +32,8 @@ void test_conv2d(ComputeGraph& graph, const std::vector& args) { // impl_selector grammar: // "" -> aten.convolution.default (heuristic-routed: // should_use_conv2d_im2col() picks direct vs im2col) - // "direct" -> add_conv2d_node(force_direct=true): forces the direct - // sliding-window path, bypassing the routing heuristic + // "direct" -> conv2d_direct_impl: forces the direct sliding-window + // path, bypassing the routing heuristic // "im2col" -> et_vk.conv2d_gemm.default, auto im2col storage // "im2col_buffer"-> im2col/GEMM, force buffer im2col intermediate // "im2col_tex2d" -> im2col/GEMM, force texture2d im2col intermediate @@ -94,11 +94,10 @@ void test_conv2d(ComputeGraph& graph, const std::vector& args) { // The "direct" selector must reach the exact direct sliding-window dispatch // the heuristic would otherwise pick. The registered op can only route via - // the heuristic, so call add_conv2d_node directly with force_direct=true to - // bypass it (mirroring how the forced-storage variants call - // conv2d_gemm_impl). + // the heuristic, so call conv2d_direct_impl directly to bypass it (mirroring + // how the forced-storage variants call conv2d_gemm_impl). if (impl_selector == "direct") { - add_conv2d_node( + conv2d_direct_impl( graph, input, weight, @@ -109,11 +108,10 @@ void test_conv2d(ComputeGraph& graph, const std::vector& args) { transposed, output_padding, groups, - /*out_min=*/kDummyValueRef, - /*out_max=*/kDummyValueRef, out, /*clamp_out=*/false, - /*force_direct=*/true); + /*out_min_val=*/0.0f, + /*out_max_val=*/0.0f); return; } diff --git a/backends/vulkan/test/custom_ops/impl/TestConv2dDw.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2dDw.cpp similarity index 98% rename from backends/vulkan/test/custom_ops/impl/TestConv2dDw.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2dDw.cpp index 85c2308b11f..f27a596eca3 100644 --- a/backends/vulkan/test/custom_ops/impl/TestConv2dDw.cpp +++ b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2dDw.cpp @@ -9,8 +9,9 @@ #include #include -#include #include +#include +#include #include #include diff --git a/backends/vulkan/test/custom_ops/impl/TestConv2dPw.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2dPw.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestConv2dPw.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestConv2dPw.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestFpaQ4gswLinear.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestFpaQ4gswLinear.cpp similarity index 98% rename from backends/vulkan/test/custom_ops/impl/TestFpaQ4gswLinear.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestFpaQ4gswLinear.cpp index a9e5ad96e95..4b41391b2cd 100644 --- a/backends/vulkan/test/custom_ops/impl/TestFpaQ4gswLinear.cpp +++ b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestFpaQ4gswLinear.cpp @@ -10,9 +10,9 @@ #include #include -#include #include #include +#include #include namespace vkcompute { @@ -212,10 +212,10 @@ LocalWorkGroup pick_q4gsw_coop_lwg( } // Spec-constant LWG values for the q4gsw_linear_gemv__w_4x8_nc[_nosg] shaders -// (now shipped from test/custom_ops/glsl/). The sg variant pins subgroupSize -// to 64 via VK_EXT_subgroup_size_control; the nosg variant uses shared-mem -// reduction so the lane count is purely an LWG choice. Both use 4 subgroups -// per workgroup. +// (now shipped from test/ops/benchmark/test_ops/glsl/). The sg variant pins +// subgroupSize to 64 via VK_EXT_subgroup_size_control; the nosg variant uses +// shared-mem reduction so the lane count is purely an LWG choice. Both use 4 +// subgroups per workgroup. constexpr uint32_t kGemvSubgroupSize = 64u; constexpr uint32_t kGemvNumSubgroups = 4u; diff --git a/backends/vulkan/test/custom_ops/impl/TestMatmulLinear.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestMatmulLinear.cpp similarity index 92% rename from backends/vulkan/test/custom_ops/impl/TestMatmulLinear.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestMatmulLinear.cpp index 60eeb2b034c..fc5fb20cab4 100644 --- a/backends/vulkan/test/custom_ops/impl/TestMatmulLinear.cpp +++ b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestMatmulLinear.cpp @@ -9,11 +9,12 @@ #include #include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include #include namespace vkcompute { diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taBinary.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taBinary.cpp similarity index 97% rename from backends/vulkan/test/custom_ops/impl/TestQ8taBinary.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taBinary.cpp index e3c3e6e2642..00b042c8180 100644 --- a/backends/vulkan/test/custom_ops/impl/TestQ8taBinary.cpp +++ b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taBinary.cpp @@ -9,8 +9,8 @@ #include #include -#include #include +#include namespace vkcompute { diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taClone.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taClone.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taClone.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taClone.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taConv2d.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taConv2d.cpp similarity index 98% rename from backends/vulkan/test/custom_ops/impl/TestQ8taConv2d.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taConv2d.cpp index e4af2c56b9d..4da454040e7 100644 --- a/backends/vulkan/test/custom_ops/impl/TestQ8taConv2d.cpp +++ b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taConv2d.cpp @@ -10,8 +10,9 @@ #include #include -#include #include +#include +#include namespace vkcompute { diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taConv2dTransposed.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taConv2dTransposed.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taConv2dTransposed.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taConv2dTransposed.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taLinear.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taLinear.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taLinear.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taLinear.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taPixelShuffle.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taPixelShuffle.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taPixelShuffle.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taPixelShuffle.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taQuantizeDequantize.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taQuantizeDequantize.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taQuantizeDequantize.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taQuantizeDequantize.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestQ8taUnary.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taUnary.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestQ8taUnary.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestQ8taUnary.cpp diff --git a/backends/vulkan/test/custom_ops/impl/TestSDPA.cpp b/backends/vulkan/test/ops/benchmark/test_ops/impl/TestSDPA.cpp similarity index 100% rename from backends/vulkan/test/custom_ops/impl/TestSDPA.cpp rename to backends/vulkan/test/ops/benchmark/test_ops/impl/TestSDPA.cpp diff --git a/backends/vulkan/test/spv_codegen/test_gen_vulkan_spv.py b/backends/vulkan/test/spv_codegen/test_gen_vulkan_spv.py new file mode 100644 index 00000000000..5a5d78ee5b5 --- /dev/null +++ b/backends/vulkan/test/spv_codegen/test_gen_vulkan_spv.py @@ -0,0 +1,262 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import stat +import tempfile +import unittest +from pathlib import Path + +from runtime import gen_vulkan_spv + + +class TestNestedShaderSources(unittest.TestCase): + def _write_fake_compiler( + self, directory: Path, assertions: str = "" + ) -> tuple[Path, Path]: + compiler_log = directory / "compiler.log" + compiler = directory / "fake_glslc.py" + compiler.write_text( + "#!/usr/bin/env python3\n" + "import pathlib\n" + "import sys\n" + "log = pathlib.Path(__file__).with_name('compiler.log')\n" + "with log.open('a') as log_file:\n" + " log_file.write('compile\\n')\n" + "source = pathlib.Path(sys.argv[2])\n" + + assertions + + "output = pathlib.Path(sys.argv[sys.argv.index('-o') + 1])\n" + "output.write_bytes(b'\\x03\\x02\\x23\\x07')\n" + ) + compiler.chmod(compiler.stat().st_mode | stat.S_IXUSR) + return compiler, compiler_log + + def _write_shader(self, directory: Path, variant_name: str) -> None: + directory.mkdir(parents=True) + (directory / "common.glslh").write_text("const uint value = 1;\n") + (directory / "kernel.glsl").write_text( + '#version 450\n#include "common.glslh"\nvoid main() {}\n' + ) + (directory / "kernel.yaml").write_text( + "kernel:\n" + " parameter_names_with_default_values: {}\n" + " shader_variants:\n" + f" - NAME: {variant_name}\n" + ) + + def test_repeated_template_names_in_nested_directories(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + self._write_shader(source_root / "direct", "direct_kernel") + self._write_shader(source_root / "winograd", "winograd_kernel") + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=None + ) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + self.assertTrue((output_dir / "direct" / "direct_kernel.glsl").is_file()) + self.assertTrue((output_dir / "direct" / "common.glslh").is_file()) + self.assertTrue( + (output_dir / "winograd" / "winograd_kernel.glsl").is_file() + ) + self.assertTrue((output_dir / "winograd" / "common.glslh").is_file()) + + def test_multi_dot_source_preserves_legacy_template_name(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + source_root.mkdir() + (source_root / "kernel.special.glsl").write_text( + "#version 450\nvoid main() {}\n" + ) + (source_root / "kernel.yaml").write_text( + "kernel:\n" + " parameter_names_with_default_values: {}\n" + " shader_variants:\n" + " - NAME: generated_kernel\n" + ) + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=None + ) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + self.assertTrue((output_dir / "generated_kernel.glsl").is_file()) + + def test_rejects_generated_shader_outside_output_directory(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + self._write_shader(source_root, str(root / "escaped_kernel")) + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + compiler, _ = self._write_fake_compiler(root) + + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=str(compiler) + ) + with self.assertRaisesRegex(ValueError, "outside output directory"): + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + def test_local_include_change_recompiles_nested_shader(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + shader_dir = source_root / "nested" + self._write_shader(shader_dir, "nested_kernel") + (source_root / "root_common.glslh").write_text( + "const uint root_value = 1;\n" + ) + (shader_dir / "kernel.glsl").write_text( + '#version 450\n#include "common.glslh"\n' + '#include "root_common.glslh"\nvoid main() {}\n' + ) + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + + compiler, compiler_log = self._write_fake_compiler( + root, + "include_indices = [i for i, arg in enumerate(sys.argv) if arg == '-I']\n" + "local_include_dir = pathlib.Path(sys.argv[include_indices[0] + 1])\n" + "root_include_dir = pathlib.Path(sys.argv[include_indices[1] + 1])\n" + "assert local_include_dir == source.parent\n" + "assert (local_include_dir / 'common.glslh').is_file()\n" + "assert root_include_dir == source.parent.parent\n" + "assert (root_include_dir / 'root_common.glslh').is_file()\n", + ) + + def generate() -> None: + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=str(compiler) + ) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + generate() + generate() + self.assertEqual(compiler_log.read_text().splitlines(), ["compile"]) + + (shader_dir / "common.glslh").write_text("const uint value = 2;\n") + generate() + + self.assertEqual( + compiler_log.read_text().splitlines(), ["compile", "compile"] + ) + + (source_root / "root_common.glslh").write_text( + "const uint root_value = 2;\n" + ) + generate() + + self.assertEqual( + compiler_log.read_text().splitlines(), + ["compile", "compile", "compile"], + ) + + def test_ancestor_include_change_recompiles_nested_shader(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + family_dir = source_root / "family" + shader_dir = family_dir / "route" + self._write_shader(shader_dir, "route_kernel") + (family_dir / "family_common.glslh").write_text( + "const uint family_value = 1;\n" + ) + (shader_dir / "kernel.glsl").write_text( + '#version 450\n#include "family_common.glslh"\nvoid main() {}\n' + ) + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + + compiler, compiler_log = self._write_fake_compiler( + root, + "include_dirs = [pathlib.Path(sys.argv[i + 1]) " + "for i, arg in enumerate(sys.argv) if arg == '-I']\n" + "assert include_dirs == [source.parent, source.parent.parent, " + "source.parent.parent.parent]\n", + ) + + def generate() -> None: + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=str(compiler) + ) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + generate() + generate() + self.assertEqual(compiler_log.read_text().splitlines(), ["compile"]) + + (family_dir / "family_common.glslh").write_text( + "const uint family_value = 2;\n" + ) + generate() + + self.assertEqual( + compiler_log.read_text().splitlines(), ["compile", "compile"] + ) + + def test_untracked_include_disables_cache(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + self._write_shader(source_root, "kernel") + (source_root / "kernel.glsl").write_text( + '#version 450\n#include "external.glslh"\nvoid main() {}\n' + ) + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + compiler, compiler_log = self._write_fake_compiler(root) + + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=str(compiler) + ) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + self.assertEqual( + compiler_log.read_text().splitlines(), ["compile", "compile"] + ) + + def test_cyclic_includes_use_cache(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_root = root / "src" + self._write_shader(source_root, "kernel") + (source_root / "kernel.glsl").write_text( + '#version 450\n#include "a.glslh"\nvoid main() {}\n' + ) + (source_root / "a.glslh").write_text('#include "b.glslh"\n') + (source_root / "b.glslh").write_text('#include "a.glslh"\n') + output_dir = root / "out" + cache_dir = root / "cache" + output_dir.mkdir() + cache_dir.mkdir() + compiler, compiler_log = self._write_fake_compiler(root) + + generator = gen_vulkan_spv.SPVGenerator( + str(source_root), {}, glslc_path=str(compiler) + ) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + generator.generateSPV(output_dir, cache_dir, nthreads=1) + + self.assertEqual(compiler_log.read_text().splitlines(), ["compile"]) diff --git a/backends/vulkan/test/targets.bzl b/backends/vulkan/test/targets.bzl index 0051a74a530..e3b735137d5 100644 --- a/backends/vulkan/test/targets.bzl +++ b/backends/vulkan/test/targets.bzl @@ -98,6 +98,16 @@ def define_common_targets(is_fbcode = False): ], ) + python_unittest( + name = "test_gen_vulkan_spv", + srcs = [ + "spv_codegen/test_gen_vulkan_spv.py", + ], + deps = [ + "//executorch/backends/vulkan:gen_vulkan_spv_lib", + ], + ) + python_unittest( name = "test_serialization", srcs = [ @@ -164,7 +174,7 @@ def define_common_targets(is_fbcode = False): fb_native.filegroup( name = "test_shaders", srcs = glob([ - "glsl/*", + "glsl/**/*", ]), visibility = [ "PUBLIC", diff --git a/backends/vulkan/test/test_vulkan_shader_names.py b/backends/vulkan/test/test_vulkan_shader_names.py index 1535e6c9653..d7843a075e2 100644 --- a/backends/vulkan/test/test_vulkan_shader_names.py +++ b/backends/vulkan/test/test_vulkan_shader_names.py @@ -24,6 +24,7 @@ """ import importlib.util +import os import re import unittest from pathlib import Path @@ -42,7 +43,12 @@ _spec.loader.exec_module(gen_vulkan_spv) # The yaml templates whose variants `add_binary_op_node` dispatches into. -_BINARY_TEMPLATES = ("binary_op_buffer", "binary_op_texture") +_BINARY_BUFFER_TEMPLATE = os.path.join("binary", "binary_op_buffer") +_BINARY_TEXTURE_TEMPLATE = os.path.join("binary", "binary_op_texture") +_BINARY_TEMPLATES = ( + _BINARY_BUFFER_TEMPLATE, + _BINARY_TEXTURE_TEMPLATE, +) def _function_body(text: str, name: str) -> str: @@ -113,8 +119,8 @@ def test_int32_eq_shaders_are_named_for_the_dispatcher(self) -> None: build. Kokoro's synthesizer aborted on it. """ for template, expected in ( - ("binary_op_buffer", "binary_eq_buffer_int32"), - ("binary_op_texture", "binary_eq_texture3d_int32"), + (_BINARY_BUFFER_TEMPLATE, "binary_eq_buffer_int32"), + (_BINARY_TEXTURE_TEMPLATE, "binary_eq_texture3d_int32"), ): with self.subTest(template=template): self.assertIn(expected, self.names[template]) diff --git a/backends/vulkan/test/vulkan_compute_api_test.cpp b/backends/vulkan/test/vulkan_compute_api_test.cpp index d77b8e8c95c..d56fee3713a 100644 --- a/backends/vulkan/test/vulkan_compute_api_test.cpp +++ b/backends/vulkan/test/vulkan_compute_api_test.cpp @@ -4316,15 +4316,35 @@ TEST_P(VulkanSwiGLUTest, MatchesUnfusedAcrossResizes) { const float y = float(up_data[broadcast ? i % width : i]); const float reference = (x / (1.0f + std::exp(-x))) * y; if (dtype == vkapi::kHalf) { + constexpr float fp16_min_normal = 0x1p-14f; + constexpr float unfused_rtol = 3e-3f; ASSERT_NEAR( - float(actual[i]), reference, 1e-3f * std::abs(reference) + 1e-7f) + float(actual[i]), + reference, + 1e-3f * std::abs(reference) + fp16_min_normal) << "element " << i; - // The unfused path rounds twice more, including subnormal sigmoid. - const float atol = 6e-8f * (std::abs(x * y) + std::abs(y) + 1.0f); + const float sigmoid_reference = 1.0f / (1.0f + std::exp(-x)); + const auto is_subnormal = [](const float value) { + return value != 0.0f && std::abs(value) < fp16_min_normal; + }; + const auto may_underflow = [](const float value) { + return value != 0.0f && + std::abs(value) < fp16_min_normal * (1.0f + unfused_rtol); + }; + float atol = 6e-8f * (std::abs(x * y) + std::abs(y) + 1.0f); + if (is_subnormal(x) || is_subnormal(y) || + may_underflow(sigmoid_reference) || + may_underflow(x * sigmoid_reference) || + may_underflow(reference)) { + // Flushing the unfused path to zero can lose the full reference + // magnitude. Its relative tolerance then vanishes, so the extra + // 1e-3 covers the fused result's relative error. + atol += (1.0f + 1e-3f) * std::abs(reference); + } ASSERT_NEAR( float(actual[i]), float(expected[i]), - 3e-3f * std::abs(float(expected[i])) + atol) + unfused_rtol * std::abs(float(expected[i])) + atol) << "element " << i; } else { ASSERT_NEAR( diff --git a/backends/vulkan/tools/gpuinfo/targets.bzl b/backends/vulkan/tools/gpuinfo/targets.bzl index 2bc9b3f1d04..be0a9680609 100644 --- a/backends/vulkan/tools/gpuinfo/targets.bzl +++ b/backends/vulkan/tools/gpuinfo/targets.bzl @@ -8,7 +8,7 @@ def define_common_targets(is_fbcode = False): buck_filegroup( name = "gpuinfo_shaders", srcs = native.glob([ - "glsl/*", + "glsl/**/*", ]), visibility = [ "PUBLIC",