diff --git a/csrc/engine/infer_engine.cpp b/csrc/engine/infer_engine.cpp index bc057544b..b26608c16 100644 --- a/csrc/engine/infer_engine.cpp +++ b/csrc/engine/infer_engine.cpp @@ -76,9 +76,10 @@ InferEngine::InferEngine( && device_type != infinicore::Device::Type::kMoore && device_type != infinicore::Device::Type::kCambricon && device_type != infinicore::Device::Type::kIluvatar - && device_type != infinicore::Device::Type::kHygon) { + && device_type != infinicore::Device::Type::kHygon + && device_type != infinicore::Device::Type::kAscend) { throw std::invalid_argument( - "flash-attn is only available on NVIDIA, MetaX, Moore, Cambricon, Iluvatar, and Hygon devices"); + "flash-attn is only available on NVIDIA, MetaX, Moore, Cambricon, Iluvatar, Hygon, and Ascend devices"); } const auto *paged_cache_config = dynamic_cast(cache_config); if (paged_cache_config == nullptr) { diff --git a/csrc/infinicore/src/ops/mha_kvcache/mha_kvcache_infiniops.cc b/csrc/infinicore/src/ops/mha_kvcache/mha_kvcache_infiniops.cc index 399c08fdd..e906ef119 100644 --- a/csrc/infinicore/src/ops/mha_kvcache/mha_kvcache_infiniops.cc +++ b/csrc/infinicore/src/ops/mha_kvcache/mha_kvcache_infiniops.cc @@ -18,7 +18,8 @@ using TensorMeta = ::infinicore::op::infiniops::TensorMeta; // InfiniOps provides device-aware default selection for these operators. std::size_t implementation_index_for_device( infini::ops::Device::Type device_type) { - if (device_type == infini::ops::Device::Type::kIluvatar) { + if (device_type == infini::ops::Device::Type::kIluvatar + || device_type == infini::ops::Device::Type::kAscend) { return 0; } if (device_type == infini::ops::Device::Type::kMoore) { @@ -41,7 +42,8 @@ bool is_supported(const Tensor &out, && device_type != Device::Type::kMoore && device_type != Device::Type::kCambricon && device_type != Device::Type::kIluvatar - && device_type != Device::Type::kHygon) + && device_type != Device::Type::kHygon + && device_type != Device::Type::kAscend) || q->ndim() != 4 || out->ndim() != 4 || k_cache->ndim() != 4 @@ -214,6 +216,9 @@ static bool registered = []() { MhaKVCache::plan_dispatcher().registerDevice(Device::Type::kHygon, &plan); MhaKVCache::run_dispatcher().registerDevice(Device::Type::kHygon, &run); MhaKVCache::cleanup_dispatcher().registerDevice(Device::Type::kHygon, &cleanup); + MhaKVCache::plan_dispatcher().registerDevice(Device::Type::kAscend, &plan); + MhaKVCache::run_dispatcher().registerDevice(Device::Type::kAscend, &run); + MhaKVCache::cleanup_dispatcher().registerDevice(Device::Type::kAscend, &cleanup); return true; }(); diff --git a/csrc/infinicore/src/ops/multi_head_attention_varlen/mha_varlen_infiniops.cc b/csrc/infinicore/src/ops/multi_head_attention_varlen/mha_varlen_infiniops.cc index ac098ec1f..043ce070a 100644 --- a/csrc/infinicore/src/ops/multi_head_attention_varlen/mha_varlen_infiniops.cc +++ b/csrc/infinicore/src/ops/multi_head_attention_varlen/mha_varlen_infiniops.cc @@ -18,7 +18,8 @@ using TensorMeta = ::infinicore::op::infiniops::TensorMeta; // InfiniOps provides device-aware default selection for these operators. std::size_t implementation_index_for_device( infini::ops::Device::Type device_type) { - if (device_type == infini::ops::Device::Type::kIluvatar) { + if (device_type == infini::ops::Device::Type::kIluvatar + || device_type == infini::ops::Device::Type::kAscend) { return 0; } if (device_type == infini::ops::Device::Type::kMoore) { @@ -45,7 +46,8 @@ bool is_supported(const Tensor &out, && device_type != Device::Type::kMoore && device_type != Device::Type::kCambricon && device_type != Device::Type::kIluvatar - && device_type != Device::Type::kHygon) + && device_type != Device::Type::kHygon + && device_type != Device::Type::kAscend) || q->ndim() != 3 || out->ndim() != 3 || ((paged && (k->ndim() != 4 || v->ndim() != 4)) @@ -258,6 +260,12 @@ static bool registered = []() { Device::Type::kHygon, &run); MultiheadAttentionVarlen::cleanup_dispatcher().registerDevice( Device::Type::kHygon, &cleanup); + MultiheadAttentionVarlen::plan_dispatcher().registerDevice( + Device::Type::kAscend, &plan); + MultiheadAttentionVarlen::run_dispatcher().registerDevice( + Device::Type::kAscend, &run); + MultiheadAttentionVarlen::cleanup_dispatcher().registerDevice( + Device::Type::kAscend, &cleanup); return true; }(); diff --git a/test/static/test_infinicore_runtime_contracts.py b/test/static/test_infinicore_runtime_contracts.py index 5f508881c..e687b0abb 100644 --- a/test/static/test_infinicore_runtime_contracts.py +++ b/test/static/test_infinicore_runtime_contracts.py @@ -52,9 +52,9 @@ def test_exact_infiniops_adapter_set_is_registered(self) -> None: "gelu/gelu_infiniops.cc": 1, "gelutanh/gelutanh_infiniops.cc": 1, "gemm/gemm_infiniops.cc": 3, - "mha_kvcache/mha_kvcache_infiniops.cc": 18, + "mha_kvcache/mha_kvcache_infiniops.cc": 21, "mul/mul_infiniops.cc": 3, - "multi_head_attention_varlen/mha_varlen_infiniops.cc": 18, + "multi_head_attention_varlen/mha_varlen_infiniops.cc": 21, "ones/ones_infiniops.cc": 3, "paged_caching/paged_caching_infiniops.cc": 3, "random_sample/random_sample_infiniops.cc": 1, @@ -324,6 +324,7 @@ def test_unavailable_configurable_surfaces_are_rejected(self) -> None: self.assertIn( "device_type != infinicore::Device::Type::kCambricon", infer_engine ) + self.assertIn("device_type != infinicore::Device::Type::kAscend", infer_engine) self.assertIn("paged_cache_config->num_blocks() == 0", infer_engine) self.assertIn("paged_cache_config->block_size() == 0", infer_engine) self.assertIn("paged_cache_config->block_size() % 256 != 0", infer_engine) diff --git a/test/static/test_modern_infinicore_compatibility.py b/test/static/test_modern_infinicore_compatibility.py index f2bf4c72a..29988fbb5 100644 --- a/test/static/test_modern_infinicore_compatibility.py +++ b/test/static/test_modern_infinicore_compatibility.py @@ -210,6 +210,7 @@ def test_current_infiniops_backend_bridges_are_registered(self) -> None: "kCambricon", "kIluvatar", "kHygon", + "kAscend", ): with self.subTest(adapter=relative_path, device=device): self.assertIn(f"device_type != Device::Type::{device}", attention) @@ -219,6 +220,10 @@ def test_current_infiniops_backend_bridges_are_registered(self) -> None: ) self.assertIn("configForImplementation<", attention) self.assertIn("implementation_index_for_device(device_type)", attention) + self.assertIn( + "device_type == infini::ops::Device::Type::kAscend", + attention, + ) self.assertIn("device_type == Device::Type::kMoore", attention) self.assertIn("device_type == Device::Type::kIluvatar", attention) self.assertIn("return 0;", attention) @@ -258,7 +263,11 @@ def test_current_infiniops_backend_bridges_are_registered(self) -> None: engine, ) self.assertIn( - "flash-attn is only available on NVIDIA, MetaX, Moore, Cambricon, Iluvatar, and Hygon devices", + "device_type != infinicore::Device::Type::kAscend", + engine, + ) + self.assertIn( + "flash-attn is only available on NVIDIA, MetaX, Moore, Cambricon, Iluvatar, Hygon, and Ascend devices", engine, ) self.assertIn(