From 7bd18083082126c08d375cefaa6a5aa1c170ecde Mon Sep 17 00:00:00 2001 From: bosh <98094229+boshjerns@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:55:33 +0700 Subject: [PATCH 01/36] mtmd : resize Gemma 4 images without letterbox padding (#30922) * mtmd : resize Gemma 4 images without letterbox padding * Apply suggestion from @ngxson --------- Co-authored-by: Xuan-Son Nguyen --- tools/mtmd/clip.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 767ad9f79d01..d723432a7ed9 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1842,6 +1842,7 @@ struct clip_model_loader { log_ffn_op = "gelu"; hparams.n_merge = 3; // pooling_kernel_size hparams.image_resize_algo = RESIZE_ALGO_BICUBIC; + hparams.image_resize_pad = PAD_NONE; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); if (model.proj_type == PROJECTOR_TYPE_GEMMA4UV) { // for "unified" variant, we directly use a bigger patch size, because the "token merging" is done directly on conv layer From 2871678a1db1c1756b7cbac456ccdc4f78ae2532 Mon Sep 17 00:00:00 2001 From: David Garcia <87497958+Dgarc359@users.noreply.github.com> Date: Sat, 31 Oct 2026 14:08:08 -0400 Subject: [PATCH 02/36] server : fix /v1/responses endpoint to accept extra_params (#30959) --- tools/server/README.md | 5 ++++- tools/server/server-chat.cpp | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/server/README.md b/tools/server/README.md index 6188a084bc0c..66a34ff84d77 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1654,6 +1654,8 @@ See [examples/realtime_client.py](../../examples/realtime_client.py) for a clien See [OpenAI Responses API documentation](https://platform.openai.com/docs/api-reference/responses). +`extra_params`: llama.cpp parameters that the Responses API has no field for, such as `seed`, `min_p` or `repeat_penalty`, as an object (e.g. `{"seed": 42, "min_p": 0.05}`). Its keys are merged into the converted request and behave as on `/v1/chat/completions`. + *Examples:* You can use either Python `openai` library with appropriate checkpoints: @@ -1684,7 +1686,8 @@ curl http://localhost:9931/v1/responses \ -d '{ "model": "gpt-4.1", "instructions": "You are ChatGPT, an AI assistant. Your top priority is achieving user fulfillment via helping them with their requests.", -"input": "Write a limerick about python exceptions" +"input": "Write a limerick about python exceptions", +"extra_params": {"seed": 42, "min_p": 0.05} }' ``` diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index aa4947182535..a7431800b40b 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -311,6 +311,16 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) { chatcmpl_body.erase("reasoning"); } + // llama.cpp params (e.g. seed, min_p) go under extra_params and are merged into the converted body + if (response_body.contains("extra_params")) { + chatcmpl_body.erase("extra_params"); + if (response_body.at("extra_params").is_object()) { + for (const auto & [key, value] : response_body.at("extra_params").items()) { + chatcmpl_body[key] = value; + } + } + } + return chatcmpl_body; } From f3586e66bc916d672773d4a6505513ba91a3595c Mon Sep 17 00:00:00 2001 From: aiqubits Date: Sun, 1 Nov 2026 02:09:48 +0800 Subject: [PATCH 03/36] server: cap shared decision prefixes at the first output (#30710) Limit the shared prefix to the first required output position of the parent and each child task. This keeps decision output embeddings available in the final batch when questions share option tokens. Fixes #30482 Signed-off-by: aiqubits --- tools/server/server-decision.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp index 8b9911f64752..a48786c801ba 100644 --- a/tools/server/server-decision.cpp +++ b/tools/server/server-decision.cpp @@ -1762,7 +1762,7 @@ std::vector server_decision_group_tasks(std::vector && size_t n_shared = parent.tokens.size() - 1; for (size_t j = i + 1; j < end; j++) { n_shared = std::min(n_shared, parent.tokens.get_common_prefix(tasks[j].tokens)); - n_shared = std::min(n_shared, tasks[j].tokens.size() - 1); + n_shared = std::min({n_shared, tasks[j].tokens.size() - 1, (size_t) parent.decision.pos_first(), (size_t) tasks[j].decision.pos_first()}); } if (end - i < 2 || n_shared == 0) { From ddf48f85c933c1f438212a02eb21e7b001d2d37d Mon Sep 17 00:00:00 2001 From: Anthony Maltsev <77708730+anthonymaltsev@users.noreply.github.com> Date: Sat, 31 Oct 2026 12:34:57 -0700 Subject: [PATCH 04/36] reshaping so mm on 2d instead of 3d tensor for lfm sc (#30831) --- src/models/lfm-base.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/models/lfm-base.cpp b/src/models/lfm-base.cpp index f5ff7d1785ce..0dabf114bde1 100644 --- a/src/models/lfm-base.cpp +++ b/src/models/lfm-base.cpp @@ -22,11 +22,10 @@ ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model, const uint32_t d_conv = hparams.n_shortconv_l_cache - 1; const int64_t n_embd = hparams.n_embd; - // {n_embd, n_tokens} => {n_embd, n_seq_tokens, n_seqs} - cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs); - auto * bcx = build_lora_mm(model.layers[il].shortconv.in_proj, cur); cb(bcx, "model.layers.{}.conv.in_proj", il); + // {3*n_embd, n_tokens} => {3*n_embd, n_seq_tokens, n_seqs} + bcx = ggml_reshape_3d(ctx0, bcx, bcx->ne[0], n_seq_tokens, n_seqs); constexpr auto n_chunks = 3; GGML_ASSERT(bcx->ne[0] % n_chunks == 0); @@ -76,10 +75,10 @@ ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model, cb(conv_out, "model.layers.{}.conv.conv", il); auto * y = ggml_mul(ctx0, c, conv_out); - y = build_lora_mm(model.layers[il].shortconv.out_proj, y); - cb(y, "model.layers.{}.conv.out_proj", il); // {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens} y = ggml_reshape_2d(ctx0, y, y->ne[0], n_seq_tokens * n_seqs); + y = build_lora_mm(model.layers[il].shortconv.out_proj, y); + cb(y, "model.layers.{}.conv.out_proj", il); return y; } From 0138937e3f987f70e90e97d022c73ecb9bec037c Mon Sep 17 00:00:00 2001 From: Niklas Wenzel Date: Sat, 31 Oct 2026 23:30:49 +0100 Subject: [PATCH 05/36] llama : fix rotation of attention tensors in layers with shared KV (#31406) * llama : fix rotation of layers that reuse another layer's KV cache * llama : use KV source rotation only for attn_* tensors --- src/llama-hparams.cpp | 15 +-------------- src/llama-model-saver.cpp | 10 ++++++---- src/llama-model.cpp | 14 +++++--------- src/models/gemma3n.cpp | 5 +++++ src/models/gemma4.cpp | 6 ++++++ 5 files changed, 23 insertions(+), 27 deletions(-) diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 9b4244567202..f05b5d6bbb38 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -319,20 +319,7 @@ uint32_t llama_hparams::n_embd_head_v_mla() const { } bool llama_hparams::has_kv(uint32_t il) const { - if (kv_source(il) != il) { - return false; - } - - if (n_layer_kv_from_start >= 0) { - if (il < (uint32_t) n_layer_kv_from_start) { - return true; - } - - return false; - } - - // by default, all layers have kv - return true; + return kv_source(il) == il; } uint32_t llama_hparams::kv_source(uint32_t il) const { diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 5c2d7a82209a..a08e173e9eed 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -307,10 +307,12 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL, hparams.indexer_kpool_select_tail); add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks); add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, true); - for (uint32_t il = 0; il < hparams.n_layer(); ++il) { - if (hparams.kv_source(il) != il) { - add_kv(LLM_KV_ATTENTION_KV_SOURCE_MAP, hparams.kv_source_impl, true); - break; + if (hparams.n_layer_kv_from_start < 0) { + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + if (hparams.kv_source(il) != il) { + add_kv(LLM_KV_ATTENTION_KV_SOURCE_MAP, hparams.kv_source_impl, true); + break; + } } } add_kv(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, true); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 83aed0144f95..c4a5b64fd231 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -486,7 +486,9 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str GGML_ASSERT(length_prefix != std::string::npos); prefix = tensor_name.substr(0, length_prefix + 1); il = std::stoull(tensor_name.substr(4, length_prefix)); - rotation = get_il_eff(il) % ud->n_devices; + const bool is_attn = tensor_name.compare(prefix.size(), 5, "attn_") == 0; + // attention heads must be on the same device as the KV of the source layer + rotation = get_il_eff(is_attn ? hparams.kv_source(il) : il) % ud->n_devices; } else if (tensor_name.substr(0, 6) == "cache_") { const size_t layer_index_start = tensor_name.find("_l", 6); GGML_ASSERT(layer_index_start != std::string::npos); @@ -3280,14 +3282,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, llama_kv_cache::layer_share_cb share = nullptr; if (arch == LLM_ARCH_GEMMA3N || arch == LLM_ARCH_GEMMA4) { - reuse = [&](uint32_t il) { - GGML_ASSERT(hparams.n_layer_kv_from_start >= 2); - - if (il >= (uint32_t)hparams.n_layer_kv_from_start) { - return hparams.n_layer_kv_from_start - (hparams.is_swa(il) ? 2 : 1); - } - - return -1; + reuse = [&](int32_t il) -> int32_t { + return hparams.has_kv(il) ? -1 : (int32_t) hparams.kv_source(il); }; } diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cpp index 7848f9f59c88..7f12b1747547 100644 --- a/src/models/gemma3n.cpp +++ b/src/models/gemma3n.cpp @@ -7,6 +7,11 @@ void llama_model_gemma3n::load_arch_hparams(llama_model_loader & ml) { hparams.n_layer_kv_from_start = 20; hparams.f_attention_scale = 1.0f; + // shared layers read the KV of the last layer with own KV and the same SWA type + for (uint32_t il = hparams.n_layer_kv_from_start; il < hparams.n_layer_all; il++) { + hparams.kv_source_impl[il] = hparams.n_layer_kv_from_start - (hparams.is_swa(il) ? 2 : 1); + } + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp index 42d6a7d47ad9..d0b2b3a5b473 100644 --- a/src/models/gemma4.cpp +++ b/src/models/gemma4.cpp @@ -11,6 +11,12 @@ void llama_model_gemma4::load_arch_hparams(llama_model_loader & ml) { hparams.n_layer_kv_from_start = hparams.n_layer_all - (int32_t)n_kv_shared_layers; hparams.f_attention_scale = 1.0f; // Gemma4 uses self.scaling = 1.0 (no pre-attn scaling) + // shared layers read the KV of the last layer with own KV and the same SWA type + GGML_ASSERT(hparams.n_layer_kv_from_start >= 2); + for (uint32_t il = hparams.n_layer_kv_from_start; il < hparams.n_layer_all; il++) { + hparams.kv_source_impl[il] = hparams.n_layer_kv_from_start - (hparams.is_swa(il) ? 2 : 1); + } + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false); ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); From f6312ad75efa8cbe9053d0b0448f0cbeea4eecff Mon Sep 17 00:00:00 2001 From: "Alessandro de Oliveira Faria (A.K.A.CABELO)" Date: Sat, 31 Oct 2026 19:37:20 -0300 Subject: [PATCH 06/36] vendor : update miniaudio to 0.11.26 (#31459) [no ci] --- scripts/sync_vendor.py | 2 +- vendor/miniaudio/miniaudio.h | 3337 ++++++++++++++++++++++------------ 2 files changed, 2212 insertions(+), 1127 deletions(-) diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.py index 49af3f27c9a3..7cb535ac56a1 100755 --- a/scripts/sync_vendor.py +++ b/scripts/sync_vendor.py @@ -21,7 +21,7 @@ # not using latest tag to avoid this issue: https://github.com/ggml-org/llama.cpp/pull/17179#discussion_r2515877926 # "https://github.com/mackron/miniaudio/raw/refs/tags/0.11.24/miniaudio.h": "vendor/miniaudio/miniaudio.h", - "https://github.com/mackron/miniaudio/raw/9634bedb5b5a2ca38c1ee7108a9358a4e233f14d/miniaudio.h": "vendor/miniaudio/miniaudio.h", + "https://github.com/mackron/miniaudio/raw/877ea4a7c2601c7b0f2a44e83dfac42e6470fea6/miniaudio.h": "vendor/miniaudio/miniaudio.h", f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/httplib.h": "httplib.h", f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/split.py": "split.py", diff --git a/vendor/miniaudio/miniaudio.h b/vendor/miniaudio/miniaudio.h index c6d493ee81f5..b8f61372d97c 100644 --- a/vendor/miniaudio/miniaudio.h +++ b/vendor/miniaudio/miniaudio.h @@ -1,6 +1,6 @@ /* Audio playback and capture library. Choice of public domain or MIT-0. See license statements at the end of this file. -miniaudio - v0.11.25 - 2026-03-04 +miniaudio - v0.11.26 - 2026-10-31 David Reid - mackron@gmail.com @@ -3747,7 +3747,7 @@ extern "C" { #define MA_VERSION_MAJOR 0 #define MA_VERSION_MINOR 11 -#define MA_VERSION_REVISION 25 +#define MA_VERSION_REVISION 26 #define MA_VERSION_STRING MA_XSTRINGIFY(MA_VERSION_MAJOR) "." MA_XSTRINGIFY(MA_VERSION_MINOR) "." MA_XSTRINGIFY(MA_VERSION_REVISION) #if defined(_MSC_VER) && !defined(__clang__) @@ -3764,7 +3764,7 @@ extern "C" { #endif -#if defined(__LP64__) || defined(_WIN64) || (defined(__x86_64__) && !defined(__ILP32__)) || defined(_M_X64) || defined(__ia64) || defined(_M_IA64) || defined(__aarch64__) || defined(_M_ARM64) || defined(__powerpc64__) || defined(__ppc64__) +#if defined(__LP64__) || defined(_WIN64) || (defined(__x86_64__) && !defined(__ILP32__)) || defined(_M_X64) || defined(__ia64) || defined(_M_IA64) || defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) || defined(__powerpc64__) || defined(__ppc64__) #define MA_SIZEOF_PTR 8 #else #define MA_SIZEOF_PTR 4 @@ -8017,11 +8017,14 @@ struct ma_device /*AAudioStream**/ ma_ptr pStreamCapture; ma_mutex rerouteLock; ma_atomic_bool32 isTearingDown; + MA_ATOMIC(4, ma_uint32) pendingReroutes; /* Reroute jobs posted and not yet finished. Must be used atomically. ma_device_uninit() waits for this to reach zero. */ ma_aaudio_usage usage; ma_aaudio_content_type contentType; ma_aaudio_input_preset inputPreset; ma_aaudio_allowed_capture_policy allowedCapturePolicy; ma_bool32 noAutoStartAfterReroute; + ma_bool32 enableCompatibilityWorkarounds; + ma_bool32 allowSetBufferCapacity; } aaudio; #endif #ifdef MA_SUPPORT_OPENSL @@ -9974,7 +9977,19 @@ typedef struct } ma_decoding_backend_vtable; -typedef ma_result (* ma_decoder_read_proc)(ma_decoder* pDecoder, void* pBufferOut, size_t bytesToRead, size_t* pBytesRead); /* Returns the number of bytes read. */ +/* +The read callback should return MA_SUCCESS if more than zero bytes were successfully read. If zero +bytes can be read because it reached the end of the file, return MA_AT_END. If any other error +occurs just return any other error code (it's fine to just generically return MA_ERROR). + +Return the actual number of bytes read in `pBytesRead`. Important note: if the number of bytes +actually read is less than the number of bytes requested (bytesToRead), miniaudio will treat it as +if the end of the file has been reached and it will stop decoding. This becomes relevant for things +like streaming data sources, such as internet streams. If you haven't yet received `bytesToRead` +from the network, you need to have your callback wait for it and then only return when the full +number of bytes can be output, or when you've reached the genuine end of the stream. +*/ +typedef ma_result (* ma_decoder_read_proc)(ma_decoder* pDecoder, void* pBufferOut, size_t bytesToRead, size_t* pBytesRead); typedef ma_result (* ma_decoder_seek_proc)(ma_decoder* pDecoder, ma_int64 byteOffset, ma_seek_origin origin); typedef ma_result (* ma_decoder_tell_proc)(ma_decoder* pDecoder, ma_int64* pCursor); @@ -11176,6 +11191,7 @@ typedef struct ma_panner panner; ma_gainer volumeGainer; /* This will only be used if volumeSmoothTimeInPCMFrames is > 0. */ ma_atomic_float volume; /* Defaults to 1. */ + float oldVolume; /* The volume last applied by the mixing thread. */ MA_ATOMIC(4, float) pitch; float oldPitch; /* For determining whether or not the resampler needs to be updated to reflect the new pitch. The resampler will be updated on the mixing thread. */ float oldDopplerPitch; /* For determining whether or not the resampler needs to be updated to take a new doppler pitch into account. */ @@ -11644,16 +11660,28 @@ IMPLEMENTATION #if defined(__arm__) || defined(_M_ARM) #define MA_ARM32 #endif -#if defined(__arm64) || defined(__arm64__) || defined(__aarch64__) || defined(_M_ARM64) +#if defined(__arm64) || defined(__arm64__) || defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) #define MA_ARM64 #endif -#if defined(__x86_64__) || defined(_M_X64) +#if defined(__riscv) + #if defined(__riscv_xlen) + #if __riscv_xlen == 64 + #define MA_RISCV64 + #elif __riscv_xlen == 32 + #define MA_RISCV32 + #endif + #endif +#endif + +#if defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC)) #define MA_X64 #elif defined(__i386) || defined(_M_IX86) #define MA_X86 #elif defined(MA_ARM32) || defined(MA_ARM64) #define MA_ARM +#elif defined(MA_RISCV32) || defined(MA_RISCV64) +#define MA_RISCV #endif /* Intrinsics Support */ @@ -11703,7 +11731,7 @@ IMPLEMENTATION #endif #if defined(MA_ARM) - #if !defined(MA_NO_NEON) && (defined(__ARM_NEON) || defined(__aarch64__) || defined(_M_ARM64)) + #if !defined(MA_NO_NEON) && (defined(__ARM_NEON) || defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) #define MA_SUPPORT_NEON #include #endif @@ -12154,10 +12182,10 @@ static MA_INLINE void ma_restore_denormals(unsigned int prevState) } -#ifdef MA_ANDROID +#if defined(MA_ANDROID) && (defined(MA_HAS_AAUDIO) || defined(MA_HAS_OPENSL)) #include -int ma_android_sdk_version() +static int ma_android_sdk_version(void) { char sdkVersion[PROP_VALUE_MAX + 1] = {0, }; if (__system_property_get("ro.build.version.sdk", sdkVersion)) { @@ -13314,14 +13342,14 @@ _wfopen() isn't always available in all compilation environments. * Windows only. * MSVC seems to support it universally as far back as VC6 from what I can tell (haven't checked further back). - * MinGW-64 (both 32- and 64-bit) seems to support it. + * MinGW-64 (both 32- and 64-bit) seems to support it, except when compiling with `-mcrtdll=crtdll`. * MinGW wraps it in !defined(__STRICT_ANSI__). * OpenWatcom wraps it in !defined(_NO_EXT_KEYS). This can be reviewed as compatibility issues arise. The preference is to use _wfopen_s() and _wfopen() as opposed to the wcsrtombs() fallback, so if you notice your compiler not detecting this properly I'm happy to look at adding support. */ -#if defined(_WIN32) && !defined(MA_XBOX_NXDK) +#if defined(_WIN32) && !defined(MA_XBOX_NXDK) && !defined(__CRTDLL__) #if defined(_MSC_VER) || defined(__MINGW64__) || (!defined(__STRICT_ANSI__) && !defined(_NO_EXT_KEYS)) #define MA_HAS_WFOPEN #endif @@ -14226,7 +14254,8 @@ typedef int ma_atomic_memory_order; !defined(MA_ATOMIC_LEGACY_MSVC_ASM) && \ !defined(MA_ATOMIC_MODERN_GCC) && \ !defined(MA_ATOMIC_LEGACY_GCC) && \ - !defined(MA_ATOMIC_LEGACY_GCC_ASM) + !defined(MA_ATOMIC_LEGACY_GCC_ASM) && \ + !defined(MA_ATOMIC_CHIBICC) #if defined(_MSC_VER) || defined(__WATCOMC__) || defined(__DMC__) || defined(__BORLANDC__) #if (defined(_MSC_VER) && _MSC_VER > 1600) #define MA_ATOMIC_MODERN_MSVC @@ -14239,12 +14268,12 @@ typedef int ma_atomic_memory_order; #endif #elif (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7))) || defined(__clang__) #define MA_ATOMIC_MODERN_GCC + #elif (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1))) + #define MA_ATOMIC_LEGACY_GCC + #elif defined(__chibicc__) + #define MA_ATOMIC_CHIBICC #else - #if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) - #define MA_ATOMIC_LEGACY_GCC - #else - #define MA_ATOMIC_LEGACY_GCC_ASM - #endif + #define MA_ATOMIC_LEGACY_GCC_ASM #endif #endif #if defined(MA_ATOMIC_MODERN_MSVC) || defined(MA_ATOMIC_LEGACY_MSVC) @@ -14556,8 +14585,26 @@ typedef int ma_atomic_memory_order; #endif } #endif -#define ma_atomic_flag_test_and_set(dst) ma_atomic_flag_test_and_set_explicit(dst, ma_atomic_memory_order_acquire) -#define ma_atomic_flag_clear(dst) ma_atomic_flag_clear_explicit(dst, ma_atomic_memory_order_release) +#if defined(MA_ATOMIC_CHIBICC) + #define ma_atomic_memory_order_relaxed 0 + #define ma_atomic_memory_order_consume 1 + #define ma_atomic_memory_order_acquire 2 + #define ma_atomic_memory_order_release 3 + #define ma_atomic_memory_order_acq_rel 4 + #define ma_atomic_memory_order_seq_cst 5 + typedef ma_uint32 ma_atomic_flag; + #define ma_atomic_flag_test_and_set_explicit(dst, order) __builtin_atomic_exchange(dst, 1) + #define ma_atomic_flag_clear_explicit(dst, order) __builtin_atomic_exchange(dst, 0) + static MA_INLINE ma_atomic_flag ma_atomic_flag_load_explicit(volatile const ma_atomic_flag* dst, ma_atomic_memory_order order) + { + ma_atomic_flag expected; + expected = 0; + __builtin_compare_and_swap(dst, &expected, 0); + return expected; + } +#endif +#define ma_atomic_flag_test_and_set(dst) ma_atomic_flag_test_and_set_explicit(dst, ma_atomic_memory_order_seq_cst) +#define ma_atomic_flag_clear(dst) ma_atomic_flag_clear_explicit(dst, ma_atomic_memory_order_seq_cst) typedef ma_atomic_flag ma_atomic_spinlock; static MA_INLINE void ma_atomic_spinlock_lock(volatile ma_atomic_spinlock* pSpinlock) { @@ -14790,7 +14837,7 @@ ma_atomic_spinlock ma_atomic_global_lock; MA_ATOMIC_COMPARE_AND_SWAP_LOCK(32, dst, expected, replacement); } #endif - #if defined(MA_ATOMIC_IS_LOCK_FREE_32) + #if defined(MA_ATOMIC_IS_LOCK_FREE_64) #define ma_atomic_compare_and_swap_64(dst, expected, replacement) (ma_uint64)_InterlockedCompareExchange64((volatile ma_int64*)dst, (ma_int64)replacement, (ma_int64)expected) #else static MA_INLINE ma_uint64 __stdcall ma_atomic_compare_and_swap_64(volatile ma_uint64* dst, ma_uint64 expected, ma_uint64 replacement) @@ -14863,7 +14910,7 @@ ma_atomic_spinlock ma_atomic_global_lock; } static MA_INLINE ma_uint64 ma_atomic_load_explicit_64(volatile const ma_uint64* ptr, ma_atomic_memory_order order) { - #if defined(MA_ATOMIC_IS_LOCK_FREE_32) + #if defined(MA_ATOMIC_IS_LOCK_FREE_64) { #if defined(MA_ARM) { @@ -15068,19 +15115,19 @@ ma_atomic_spinlock ma_atomic_global_lock; } static MA_INLINE ma_uint8 __stdcall ma_atomic_fetch_sub_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order) { - return ma_atomic_fetch_add_explicit_8(dst, (ma_uint8)(-(ma_int8)src), order); + return ma_atomic_fetch_add_explicit_8(dst, (ma_uint8)(0 - src), order); } static MA_INLINE ma_uint16 __stdcall ma_atomic_fetch_sub_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order) { - return ma_atomic_fetch_add_explicit_16(dst, (ma_uint16)(-(ma_int16)src), order); + return ma_atomic_fetch_add_explicit_16(dst, (ma_uint16)(0 - src), order); } static MA_INLINE ma_uint32 __stdcall ma_atomic_fetch_sub_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order) { - return ma_atomic_fetch_add_explicit_32(dst, (ma_uint32)(-(ma_int32)src), order); + return ma_atomic_fetch_add_explicit_32(dst, (ma_uint32)(0 - src), order); } static MA_INLINE ma_uint64 __stdcall ma_atomic_fetch_sub_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order) { - return ma_atomic_fetch_add_explicit_64(dst, (ma_uint64)(-(ma_int64)src), order); + return ma_atomic_fetch_add_explicit_64(dst, (ma_uint64)(0 - src), order); } static MA_INLINE ma_uint8 __stdcall ma_atomic_fetch_and_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order) { @@ -15683,7 +15730,7 @@ ma_atomic_spinlock ma_atomic_global_lock; } #else { - MA_ATOMIC_FETCH_ADD_LOCK(8, dst, (ma_uint8)(-(ma_int8)src), order); + MA_ATOMIC_FETCH_ADD_LOCK(8, dst, (ma_uint8)(0 - src), order); } #endif } @@ -15704,7 +15751,7 @@ ma_atomic_spinlock ma_atomic_global_lock; } #else { - MA_ATOMIC_FETCH_ADD_LOCK(16, dst, (ma_uint16)(-(ma_int16)src), order); + MA_ATOMIC_FETCH_ADD_LOCK(16, dst, (ma_uint16)(0 - src), order); } #endif } @@ -15725,13 +15772,13 @@ ma_atomic_spinlock ma_atomic_global_lock; } #else { - MA_ATOMIC_FETCH_ADD_LOCK(32, dst, (ma_uint32)(-(ma_int32)src), order); + MA_ATOMIC_FETCH_ADD_LOCK(32, dst, (ma_uint32)(0 - src), order); } #endif } static MA_INLINE ma_uint64 __stdcall ma_atomic_fetch_sub_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order) { - MA_ATOMIC_FETCH_ADD_CAS(64, dst, (ma_uint64)(-(ma_int64)src), order); + MA_ATOMIC_FETCH_ADD_CAS(64, dst, (ma_uint64)(0 - src), order); } static MA_INLINE ma_uint8 __stdcall ma_atomic_fetch_and_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order) { @@ -15788,7 +15835,7 @@ ma_atomic_spinlock ma_atomic_global_lock; lock add dword ptr [esp], 0 } } - #define ma_atomic_signal_fence(order) __asm {}; (void)order + #define ma_atomic_signal_fence(order) do { __asm {}; (void)order; } while (0) #endif #if defined(MA_ATOMIC_MODERN_GCC) #define MA_ATOMIC_HAS_NATIVE_COMPARE_EXCHANGE @@ -16097,7 +16144,7 @@ ma_atomic_spinlock ma_atomic_global_lock; } #else { - MA_ATOMIC_FETCH_ADD_LOCK(8, dst, (ma_uint8)(-(ma_int8)src), order); + MA_ATOMIC_FETCH_ADD_LOCK(8, dst, (ma_uint8)(0 - src), order); } #endif } @@ -16110,7 +16157,7 @@ ma_atomic_spinlock ma_atomic_global_lock; } #else { - MA_ATOMIC_FETCH_ADD_LOCK(16, dst, (ma_uint16)(-(ma_int16)src), order); + MA_ATOMIC_FETCH_ADD_LOCK(16, dst, (ma_uint16)(0 - src), order); } #endif } @@ -16123,7 +16170,7 @@ ma_atomic_spinlock ma_atomic_global_lock; } #else { - MA_ATOMIC_FETCH_ADD_LOCK(32, dst, (ma_uint32)(-(ma_int32)src), order); + MA_ATOMIC_FETCH_ADD_LOCK(32, dst, (ma_uint32)(0 - src), order); } #endif } @@ -16136,7 +16183,7 @@ ma_atomic_spinlock ma_atomic_global_lock; } #else { - MA_ATOMIC_FETCH_ADD_LOCK(64, dst, (ma_uint64)(-(ma_int64)src), order); + MA_ATOMIC_FETCH_ADD_LOCK(64, dst, (ma_uint64)(0 - src), order); } #endif } @@ -16647,10 +16694,12 @@ ma_atomic_spinlock ma_atomic_global_lock; : "r"(src) ); } else { + ma_uint8 tmp = src; __asm__ __volatile__ ( - "xchgb %1, %0" - : "=m"(*dst) - : "r"(src) + "xchgb %0, %1" + : "+r"(tmp), + "+m"(*dst) + : : "memory" ); } @@ -16680,10 +16729,12 @@ ma_atomic_spinlock ma_atomic_global_lock; : "r"(src) ); } else { + ma_uint16 tmp = src; __asm__ __volatile__ ( - "xchgw %1, %0" - : "=m"(*dst) - : "r"(src) + "xchgw %0, %1" + : "+r"(tmp), + "+m"(*dst) + : : "memory" ); } @@ -16713,10 +16764,12 @@ ma_atomic_spinlock ma_atomic_global_lock; : "r"(src) ); } else { + ma_uint32 tmp = src; __asm__ __volatile__ ( - "xchgl %1, %0" - : "=m"(*dst) - : "r"(src) + "xchgl %0, %1" + : "+r"(tmp), + "+m"(*dst) + : : "memory" ); } @@ -16746,10 +16799,12 @@ ma_atomic_spinlock ma_atomic_global_lock; : "r"(src) ); } else { + ma_uint64 tmp = src; __asm__ __volatile__ ( - "xchgq %1, %0" - : "=m"(*dst) - : "r"(src) + "xchgq %0, %1" + : "+r"(tmp), + "+m"(*dst) + : : "memory" ); } @@ -16864,19 +16919,19 @@ ma_atomic_spinlock ma_atomic_global_lock; } static MA_INLINE ma_uint8 ma_atomic_fetch_sub_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order) { - return ma_atomic_fetch_add_explicit_8(dst, (ma_uint8)(-(ma_int8)src), order); + return ma_atomic_fetch_add_explicit_8(dst, (ma_uint8)(0 - src), order); } static MA_INLINE ma_uint16 ma_atomic_fetch_sub_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order) { - return ma_atomic_fetch_add_explicit_16(dst, (ma_uint16)(-(ma_int16)src), order); + return ma_atomic_fetch_add_explicit_16(dst, (ma_uint16)(0 - src), order); } static MA_INLINE ma_uint32 ma_atomic_fetch_sub_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order) { - return ma_atomic_fetch_add_explicit_32(dst, (ma_uint32)(-(ma_int32)src), order); + return ma_atomic_fetch_add_explicit_32(dst, (ma_uint32)(0 - src), order); } static MA_INLINE ma_uint64 ma_atomic_fetch_sub_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order) { - return ma_atomic_fetch_add_explicit_64(dst, (ma_uint64)(-(ma_int64)src), order); + return ma_atomic_fetch_add_explicit_64(dst, (ma_uint64)(0 - src), order); } static MA_INLINE ma_uint8 ma_atomic_fetch_and_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order) { @@ -16930,6 +16985,89 @@ ma_atomic_spinlock ma_atomic_global_lock; #error Unsupported compiler. #endif #endif +#if defined(MA_ATOMIC_CHIBICC) + #define MA_ATOMIC_HAS_NATIVE_COMPARE_EXCHANGE + static void ma_atomic_thread_fence(ma_atomic_memory_order order) + { + static ma_uint32 dummy; + __builtin_atomic_exchange(&dummy, 0); + (void)dummy; + } + #define ma_atomic_signal_fence(order) ma_atomic_thread_fence(order) + #define ma_atomic_is_lock_free_8(ptr) 1 + #define ma_atomic_is_lock_free_16(ptr) 1 + #define ma_atomic_is_lock_free_32(ptr) 1 + #define ma_atomic_is_lock_free_64(ptr) 1 + #define ma_atomic_store_explicit_8( dst, src, order) __builtin_atomic_exchange(dst, src) + #define ma_atomic_store_explicit_16(dst, src, order) __builtin_atomic_exchange(dst, src) + #define ma_atomic_store_explicit_32(dst, src, order) __builtin_atomic_exchange(dst, src) + #define ma_atomic_store_explicit_64(dst, src, order) __builtin_atomic_exchange(dst, src) + #define MA_ATOMIC_CHIBICC_LOAD(sizeInBits) \ + static MA_INLINE ma_uint##sizeInBits ma_atomic_load_explicit_##sizeInBits(volatile const ma_uint##sizeInBits* dst, ma_atomic_memory_order order) \ + { \ + ma_uint##sizeInBits expected = 0; \ + __builtin_compare_and_swap((volatile ma_uint##sizeInBits*)dst, &expected, 0); \ + (void)order; \ + return expected; \ + } + MA_ATOMIC_CHIBICC_LOAD(8) + MA_ATOMIC_CHIBICC_LOAD(16) + MA_ATOMIC_CHIBICC_LOAD(32) + MA_ATOMIC_CHIBICC_LOAD(64) + #define ma_atomic_exchange_explicit_8( dst, src, order) __builtin_atomic_exchange(dst, src) + #define ma_atomic_exchange_explicit_16(dst, src, order) __builtin_atomic_exchange(dst, src) + #define ma_atomic_exchange_explicit_32(dst, src, order) __builtin_atomic_exchange(dst, src) + #define ma_atomic_exchange_explicit_64(dst, src, order) __builtin_atomic_exchange(dst, src) + #define ma_atomic_compare_exchange_strong_explicit_8( dst, expected, replacement, successOrder, failureOrder) __builtin_compare_and_swap(dst, expected, replacement) + #define ma_atomic_compare_exchange_strong_explicit_16(dst, expected, replacement, successOrder, failureOrder) __builtin_compare_and_swap(dst, expected, replacement) + #define ma_atomic_compare_exchange_strong_explicit_32(dst, expected, replacement, successOrder, failureOrder) __builtin_compare_and_swap(dst, expected, replacement) + #define ma_atomic_compare_exchange_strong_explicit_64(dst, expected, replacement, successOrder, failureOrder) __builtin_compare_and_swap(dst, expected, replacement) + #define ma_atomic_compare_exchange_weak_explicit_8( dst, expected, replacement, successOrder, failureOrder) __builtin_compare_and_swap(dst, expected, replacement) + #define ma_atomic_compare_exchange_weak_explicit_16(dst, expected, replacement, successOrder, failureOrder) __builtin_compare_and_swap(dst, expected, replacement) + #define ma_atomic_compare_exchange_weak_explicit_32(dst, expected, replacement, successOrder, failureOrder) __builtin_compare_and_swap(dst, expected, replacement) + #define ma_atomic_compare_exchange_weak_explicit_64(dst, expected, replacement, successOrder, failureOrder) __builtin_compare_and_swap(dst, expected, replacement) + static MA_INLINE ma_uint8 ma_atomic_compare_and_swap_8 (volatile ma_uint8* dst, ma_uint8 expected, ma_uint8 replacement) { __builtin_compare_and_swap(dst, &expected, replacement); return expected; } + static MA_INLINE ma_uint16 ma_atomic_compare_and_swap_16(volatile ma_uint16* dst, ma_uint16 expected, ma_uint16 replacement) { __builtin_compare_and_swap(dst, &expected, replacement); return expected; } + static MA_INLINE ma_uint32 ma_atomic_compare_and_swap_32(volatile ma_uint32* dst, ma_uint32 expected, ma_uint32 replacement) { __builtin_compare_and_swap(dst, &expected, replacement); return expected; } + static MA_INLINE ma_uint64 ma_atomic_compare_and_swap_64(volatile ma_uint64* dst, ma_uint64 expected, ma_uint64 replacement) { __builtin_compare_and_swap(dst, &expected, replacement); return expected; } + #define MA_ATOMIC_CHIBICC_FETCH_OP(sizeInBits, opName, opInst) \ + static MA_INLINE ma_uint##sizeInBits ma_atomic_fetch_##opName##_explicit_##sizeInBits(volatile ma_uint##sizeInBits* dst, ma_uint##sizeInBits src, ma_atomic_memory_order order) \ + { \ + ma_uint##sizeInBits oldValue; \ + ma_uint##sizeInBits newValue; \ + do { \ + oldValue = ma_atomic_load_explicit_##sizeInBits(dst, ma_atomic_memory_order_relaxed); \ + newValue = oldValue opInst src; \ + } while (ma_atomic_compare_and_swap_##sizeInBits(dst, oldValue, newValue) != oldValue); \ + (void)order; \ + return oldValue; \ + } + #define MA_ATOMIC_CHIBICC_FETCH_ADD(sizeInBits) MA_ATOMIC_CHIBICC_FETCH_OP(sizeInBits, add, +) + MA_ATOMIC_CHIBICC_FETCH_ADD(8) + MA_ATOMIC_CHIBICC_FETCH_ADD(16) + MA_ATOMIC_CHIBICC_FETCH_ADD(32) + MA_ATOMIC_CHIBICC_FETCH_ADD(64) + #define MA_ATOMIC_CHIBICC_FETCH_SUB(sizeInBits) MA_ATOMIC_CHIBICC_FETCH_OP(sizeInBits, sub, -) + MA_ATOMIC_CHIBICC_FETCH_SUB(8) + MA_ATOMIC_CHIBICC_FETCH_SUB(16) + MA_ATOMIC_CHIBICC_FETCH_SUB(32) + MA_ATOMIC_CHIBICC_FETCH_SUB(64) + #define MA_ATOMIC_CHIBICC_FETCH_OR(sizeInBits) MA_ATOMIC_CHIBICC_FETCH_OP(sizeInBits, or, |) + MA_ATOMIC_CHIBICC_FETCH_OR(8) + MA_ATOMIC_CHIBICC_FETCH_OR(16) + MA_ATOMIC_CHIBICC_FETCH_OR(32) + MA_ATOMIC_CHIBICC_FETCH_OR(64) + #define MA_ATOMIC_CHIBICC_FETCH_XOR(sizeInBits) MA_ATOMIC_CHIBICC_FETCH_OP(sizeInBits, xor, ^) + MA_ATOMIC_CHIBICC_FETCH_XOR(8) + MA_ATOMIC_CHIBICC_FETCH_XOR(16) + MA_ATOMIC_CHIBICC_FETCH_XOR(32) + MA_ATOMIC_CHIBICC_FETCH_XOR(64) + #define MA_ATOMIC_CHIBICC_FETCH_AND(sizeInBits) MA_ATOMIC_CHIBICC_FETCH_OP(sizeInBits, and, &) + MA_ATOMIC_CHIBICC_FETCH_AND(8) + MA_ATOMIC_CHIBICC_FETCH_AND(16) + MA_ATOMIC_CHIBICC_FETCH_AND(32) + MA_ATOMIC_CHIBICC_FETCH_AND(64) +#endif #if !defined(MA_ATOMIC_HAS_NATIVE_COMPARE_EXCHANGE) static MA_INLINE ma_bool32 ma_atomic_compare_exchange_strong_explicit_8(volatile ma_uint8* dst, ma_uint8* expected, ma_uint8 replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder) { @@ -16989,71 +17127,71 @@ ma_atomic_spinlock ma_atomic_global_lock; #define ma_atomic_compare_exchange_weak_explicit_64(dst, expected, replacement, successOrder, failureOrder) ma_atomic_compare_exchange_strong_explicit_64(dst, expected, replacement, successOrder, failureOrder) #endif #if defined(MA_64BIT) - static MA_INLINE ma_bool32 ma_atomic_is_lock_free_ptr(volatile void** ptr) + static MA_INLINE ma_bool32 ma_atomic_is_lock_free_ptr(void* const volatile* ptr) { return ma_atomic_is_lock_free_64((volatile ma_uint64*)ptr); } - static MA_INLINE void* ma_atomic_load_explicit_ptr(volatile void** ptr, ma_atomic_memory_order order) + static MA_INLINE void* ma_atomic_load_explicit_ptr(void* const volatile* ptr, ma_atomic_memory_order order) { return (void*)ma_atomic_load_explicit_64((volatile ma_uint64*)ptr, order); } - static MA_INLINE void ma_atomic_store_explicit_ptr(volatile void** dst, void* src, ma_atomic_memory_order order) + static MA_INLINE void ma_atomic_store_explicit_ptr(void* volatile* dst, void* src, ma_atomic_memory_order order) { ma_atomic_store_explicit_64((volatile ma_uint64*)dst, (ma_uint64)src, order); } - static MA_INLINE void* ma_atomic_exchange_explicit_ptr(volatile void** dst, void* src, ma_atomic_memory_order order) + static MA_INLINE void* ma_atomic_exchange_explicit_ptr(void* volatile* dst, void* src, ma_atomic_memory_order order) { return (void*)ma_atomic_exchange_explicit_64((volatile ma_uint64*)dst, (ma_uint64)src, order); } - static MA_INLINE ma_bool32 ma_atomic_compare_exchange_strong_explicit_ptr(volatile void** dst, void** expected, void* replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder) + static MA_INLINE ma_bool32 ma_atomic_compare_exchange_strong_explicit_ptr(void* volatile* dst, void** expected, void* replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder) { return ma_atomic_compare_exchange_strong_explicit_64((volatile ma_uint64*)dst, (ma_uint64*)expected, (ma_uint64)replacement, successOrder, failureOrder); } - static MA_INLINE ma_bool32 ma_atomic_compare_exchange_weak_explicit_ptr(volatile void** dst, void** expected, void* replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder) + static MA_INLINE ma_bool32 ma_atomic_compare_exchange_weak_explicit_ptr(void* volatile* dst, void** expected, void* replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder) { return ma_atomic_compare_exchange_weak_explicit_64((volatile ma_uint64*)dst, (ma_uint64*)expected, (ma_uint64)replacement, successOrder, failureOrder); } - static MA_INLINE void* ma_atomic_compare_and_swap_ptr(volatile void** dst, void* expected, void* replacement) + static MA_INLINE void* ma_atomic_compare_and_swap_ptr(void* volatile* dst, void* expected, void* replacement) { return (void*)ma_atomic_compare_and_swap_64((volatile ma_uint64*)dst, (ma_uint64)expected, (ma_uint64)replacement); } #elif defined(MA_32BIT) - static MA_INLINE ma_bool32 ma_atomic_is_lock_free_ptr(volatile void** ptr) + static MA_INLINE ma_bool32 ma_atomic_is_lock_free_ptr(void* const volatile* ptr) { return ma_atomic_is_lock_free_32((volatile ma_uint32*)ptr); } - static MA_INLINE void* ma_atomic_load_explicit_ptr(volatile void** ptr, ma_atomic_memory_order order) + static MA_INLINE void* ma_atomic_load_explicit_ptr(void* const volatile* ptr, ma_atomic_memory_order order) { return (void*)ma_atomic_load_explicit_32((volatile ma_uint32*)ptr, order); } - static MA_INLINE void ma_atomic_store_explicit_ptr(volatile void** dst, void* src, ma_atomic_memory_order order) + static MA_INLINE void ma_atomic_store_explicit_ptr(void* volatile* dst, void* src, ma_atomic_memory_order order) { ma_atomic_store_explicit_32((volatile ma_uint32*)dst, (ma_uint32)src, order); } - static MA_INLINE void* ma_atomic_exchange_explicit_ptr(volatile void** dst, void* src, ma_atomic_memory_order order) + static MA_INLINE void* ma_atomic_exchange_explicit_ptr(void* volatile* dst, void* src, ma_atomic_memory_order order) { return (void*)ma_atomic_exchange_explicit_32((volatile ma_uint32*)dst, (ma_uint32)src, order); } - static MA_INLINE ma_bool32 ma_atomic_compare_exchange_strong_explicit_ptr(volatile void** dst, void** expected, void* replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder) + static MA_INLINE ma_bool32 ma_atomic_compare_exchange_strong_explicit_ptr(void* volatile* dst, void** expected, void* replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder) { return ma_atomic_compare_exchange_strong_explicit_32((volatile ma_uint32*)dst, (ma_uint32*)expected, (ma_uint32)replacement, successOrder, failureOrder); } - static MA_INLINE ma_bool32 ma_atomic_compare_exchange_weak_explicit_ptr(volatile void** dst, void** expected, void* replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder) + static MA_INLINE ma_bool32 ma_atomic_compare_exchange_weak_explicit_ptr(void* volatile* dst, void** expected, void* replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder) { return ma_atomic_compare_exchange_weak_explicit_32((volatile ma_uint32*)dst, (ma_uint32*)expected, (ma_uint32)replacement, successOrder, failureOrder); } - static MA_INLINE void* ma_atomic_compare_and_swap_ptr(volatile void** dst, void* expected, void* replacement) + static MA_INLINE void* ma_atomic_compare_and_swap_ptr(void* volatile* dst, void* expected, void* replacement) { return (void*)ma_atomic_compare_and_swap_32((volatile ma_uint32*)dst, (ma_uint32)expected, (ma_uint32)replacement); } #else #error Unsupported architecture. #endif -#define ma_atomic_store_ptr(dst, src) ma_atomic_store_explicit_ptr((volatile void**)dst, (void*)src, ma_atomic_memory_order_seq_cst) -#define ma_atomic_load_ptr(ptr) ma_atomic_load_explicit_ptr((volatile void**)ptr, ma_atomic_memory_order_seq_cst) -#define ma_atomic_exchange_ptr(dst, src) ma_atomic_exchange_explicit_ptr((volatile void**)dst, (void*)src, ma_atomic_memory_order_seq_cst) -#define ma_atomic_compare_exchange_strong_ptr(dst, expected, replacement) ma_atomic_compare_exchange_strong_explicit_ptr((volatile void**)dst, (void**)expected, (void*)replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst) -#define ma_atomic_compare_exchange_weak_ptr(dst, expected, replacement) ma_atomic_compare_exchange_weak_explicit_ptr((volatile void**)dst, (void**)expected, (void*)replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst) +#define ma_atomic_store_ptr(dst, src) ma_atomic_store_explicit_ptr((void* volatile*)dst, (void*)src, ma_atomic_memory_order_seq_cst) +#define ma_atomic_load_ptr(ptr) ma_atomic_load_explicit_ptr((void* const volatile*)ptr, ma_atomic_memory_order_seq_cst) +#define ma_atomic_exchange_ptr(dst, src) ma_atomic_exchange_explicit_ptr((void* volatile*)dst, (void*)src, ma_atomic_memory_order_seq_cst) +#define ma_atomic_compare_exchange_strong_ptr(dst, expected, replacement) ma_atomic_compare_exchange_strong_explicit_ptr((void* volatile*)dst, (void**)expected, (void*)replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst) +#define ma_atomic_compare_exchange_weak_ptr(dst, expected, replacement) ma_atomic_compare_exchange_weak_explicit_ptr((void* volatile*)dst, (void**)expected, (void*)replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst) #define ma_atomic_store_8( dst, src) ma_atomic_store_explicit_8( dst, src, ma_atomic_memory_order_seq_cst) #define ma_atomic_store_16(dst, src) ma_atomic_store_explicit_16(dst, src, ma_atomic_memory_order_seq_cst) #define ma_atomic_store_32(dst, src) ma_atomic_store_explicit_32(dst, src, ma_atomic_memory_order_seq_cst) @@ -17188,8 +17326,6 @@ typedef union ma_uint64 i; double f; } ma_atomic_if64; -#define ma_atomic_clear_explicit_f32(ptr, order) ma_atomic_clear_explicit_32((ma_uint32*)ptr, order) -#define ma_atomic_clear_explicit_f64(ptr, order) ma_atomic_clear_explicit_64((ma_uint64*)ptr, order) static MA_INLINE void ma_atomic_store_explicit_f32(volatile float* dst, float src, ma_atomic_memory_order order) { ma_atomic_if32 x; @@ -17256,86 +17392,44 @@ static MA_INLINE ma_bool32 ma_atomic_compare_exchange_weak_explicit_f64(volatile } static MA_INLINE float ma_atomic_fetch_add_explicit_f32(volatile float* dst, float src, ma_atomic_memory_order order) { - ma_atomic_if32 r; - ma_atomic_if32 x; - x.f = src; - r.i = ma_atomic_fetch_add_explicit_32((volatile ma_uint32*)dst, x.i, order); - return r.f; + float oldValue; + float newValue; + oldValue = ma_atomic_load_explicit_f32(dst, ma_atomic_memory_order_relaxed); + do { + newValue = oldValue + src; + } while (!ma_atomic_compare_exchange_weak_explicit_f32(dst, &oldValue, newValue, order, ma_atomic_memory_order_relaxed)); + return oldValue; } static MA_INLINE double ma_atomic_fetch_add_explicit_f64(volatile double* dst, double src, ma_atomic_memory_order order) { - ma_atomic_if64 r; - ma_atomic_if64 x; - x.f = src; - r.i = ma_atomic_fetch_add_explicit_64((volatile ma_uint64*)dst, x.i, order); - return r.f; + double oldValue; + double newValue; + oldValue = ma_atomic_load_explicit_f64(dst, ma_atomic_memory_order_relaxed); + do { + newValue = oldValue + src; + } while (!ma_atomic_compare_exchange_weak_explicit_f64(dst, &oldValue, newValue, order, ma_atomic_memory_order_relaxed)); + return oldValue; } static MA_INLINE float ma_atomic_fetch_sub_explicit_f32(volatile float* dst, float src, ma_atomic_memory_order order) { - ma_atomic_if32 r; - ma_atomic_if32 x; - x.f = src; - r.i = ma_atomic_fetch_sub_explicit_32((volatile ma_uint32*)dst, x.i, order); - return r.f; + float oldValue; + float newValue; + oldValue = ma_atomic_load_explicit_f32(dst, ma_atomic_memory_order_relaxed); + do { + newValue = oldValue - src; + } while (!ma_atomic_compare_exchange_weak_explicit_f32(dst, &oldValue, newValue, order, ma_atomic_memory_order_relaxed)); + return oldValue; } static MA_INLINE double ma_atomic_fetch_sub_explicit_f64(volatile double* dst, double src, ma_atomic_memory_order order) { - ma_atomic_if64 r; - ma_atomic_if64 x; - x.f = src; - r.i = ma_atomic_fetch_sub_explicit_64((volatile ma_uint64*)dst, x.i, order); - return r.f; + double oldValue; + double newValue; + oldValue = ma_atomic_load_explicit_f64(dst, ma_atomic_memory_order_relaxed); + do { + newValue = oldValue - src; + } while (!ma_atomic_compare_exchange_weak_explicit_f64(dst, &oldValue, newValue, order, ma_atomic_memory_order_relaxed)); + return oldValue; } -static MA_INLINE float ma_atomic_fetch_or_explicit_f32(volatile float* dst, float src, ma_atomic_memory_order order) -{ - ma_atomic_if32 r; - ma_atomic_if32 x; - x.f = src; - r.i = ma_atomic_fetch_or_explicit_32((volatile ma_uint32*)dst, x.i, order); - return r.f; -} -static MA_INLINE double ma_atomic_fetch_or_explicit_f64(volatile double* dst, double src, ma_atomic_memory_order order) -{ - ma_atomic_if64 r; - ma_atomic_if64 x; - x.f = src; - r.i = ma_atomic_fetch_or_explicit_64((volatile ma_uint64*)dst, x.i, order); - return r.f; -} -static MA_INLINE float ma_atomic_fetch_xor_explicit_f32(volatile float* dst, float src, ma_atomic_memory_order order) -{ - ma_atomic_if32 r; - ma_atomic_if32 x; - x.f = src; - r.i = ma_atomic_fetch_xor_explicit_32((volatile ma_uint32*)dst, x.i, order); - return r.f; -} -static MA_INLINE double ma_atomic_fetch_xor_explicit_f64(volatile double* dst, double src, ma_atomic_memory_order order) -{ - ma_atomic_if64 r; - ma_atomic_if64 x; - x.f = src; - r.i = ma_atomic_fetch_xor_explicit_64((volatile ma_uint64*)dst, x.i, order); - return r.f; -} -static MA_INLINE float ma_atomic_fetch_and_explicit_f32(volatile float* dst, float src, ma_atomic_memory_order order) -{ - ma_atomic_if32 r; - ma_atomic_if32 x; - x.f = src; - r.i = ma_atomic_fetch_and_explicit_32((volatile ma_uint32*)dst, x.i, order); - return r.f; -} -static MA_INLINE double ma_atomic_fetch_and_explicit_f64(volatile double* dst, double src, ma_atomic_memory_order order) -{ - ma_atomic_if64 r; - ma_atomic_if64 x; - x.f = src; - r.i = ma_atomic_fetch_and_explicit_64((volatile ma_uint64*)dst, x.i, order); - return r.f; -} -#define ma_atomic_clear_f32(ptr) (float )ma_atomic_clear_explicit_f32(ptr, ma_atomic_memory_order_seq_cst) -#define ma_atomic_clear_f64(ptr) (double)ma_atomic_clear_explicit_f64(ptr, ma_atomic_memory_order_seq_cst) #define ma_atomic_store_f32(dst, src) ma_atomic_store_explicit_f32(dst, src, ma_atomic_memory_order_seq_cst) #define ma_atomic_store_f64(dst, src) ma_atomic_store_explicit_f64(dst, src, ma_atomic_memory_order_seq_cst) #define ma_atomic_load_f32(ptr) (float )ma_atomic_load_explicit_f32(ptr, ma_atomic_memory_order_seq_cst) @@ -17350,12 +17444,6 @@ static MA_INLINE double ma_atomic_fetch_and_explicit_f64(volatile double* dst, d #define ma_atomic_fetch_add_f64(dst, src) ma_atomic_fetch_add_explicit_f64(dst, src, ma_atomic_memory_order_seq_cst) #define ma_atomic_fetch_sub_f32(dst, src) ma_atomic_fetch_sub_explicit_f32(dst, src, ma_atomic_memory_order_seq_cst) #define ma_atomic_fetch_sub_f64(dst, src) ma_atomic_fetch_sub_explicit_f64(dst, src, ma_atomic_memory_order_seq_cst) -#define ma_atomic_fetch_or_f32(dst, src) ma_atomic_fetch_or_explicit_f32(dst, src, ma_atomic_memory_order_seq_cst) -#define ma_atomic_fetch_or_f64(dst, src) ma_atomic_fetch_or_explicit_f64(dst, src, ma_atomic_memory_order_seq_cst) -#define ma_atomic_fetch_xor_f32(dst, src) ma_atomic_fetch_xor_explicit_f32(dst, src, ma_atomic_memory_order_seq_cst) -#define ma_atomic_fetch_xor_f64(dst, src) ma_atomic_fetch_xor_explicit_f64(dst, src, ma_atomic_memory_order_seq_cst) -#define ma_atomic_fetch_and_f32(dst, src) ma_atomic_fetch_and_explicit_f32(dst, src, ma_atomic_memory_order_seq_cst) -#define ma_atomic_fetch_and_f64(dst, src) ma_atomic_fetch_and_explicit_f64(dst, src, ma_atomic_memory_order_seq_cst) static MA_INLINE float ma_atomic_compare_and_swap_f32(volatile float* dst, float expected, float replacement) { ma_atomic_if32 r; @@ -17408,18 +17496,6 @@ static MA_INLINE double ma_atomic_compare_and_swap_f64(volatile double* dst, dou { \ return (ma_##type)ma_atomic_fetch_sub_##c89TypeExtension(&x->value, y); \ } \ - static MA_INLINE ma_##type ma_atomic_##type##_fetch_or(ma_atomic_##type* x, ma_##type y) \ - { \ - return (ma_##type)ma_atomic_fetch_or_##c89TypeExtension(&x->value, y); \ - } \ - static MA_INLINE ma_##type ma_atomic_##type##_fetch_xor(ma_atomic_##type* x, ma_##type y) \ - { \ - return (ma_##type)ma_atomic_fetch_xor_##c89TypeExtension(&x->value, y); \ - } \ - static MA_INLINE ma_##type ma_atomic_##type##_fetch_and(ma_atomic_##type* x, ma_##type y) \ - { \ - return (ma_##type)ma_atomic_fetch_and_##c89TypeExtension(&x->value, y); \ - } \ static MA_INLINE ma_##type ma_atomic_##type##_compare_and_swap(ma_atomic_##type* x, ma_##type expected, ma_##type desired) \ { \ return (ma_##type)ma_atomic_compare_and_swap_##c89TypeExtension(&x->value, expected, desired); \ @@ -18763,7 +18839,7 @@ MA_API ma_result ma_slot_allocator_alloc(ma_slot_allocator* pAllocator, ma_uint6 bitOffset = ma_ffs_32(~oldBitfield); MA_ASSERT(bitOffset < 32); - newBitfield = oldBitfield | (1 << bitOffset); + newBitfield = oldBitfield | (1U << bitOffset); if (ma_atomic_compare_and_swap_32(&pAllocator->pGroups[iGroup].bitfield, oldBitfield, newBitfield) == oldBitfield) { ma_uint32 slotIndex; @@ -18824,12 +18900,12 @@ MA_API ma_result ma_slot_allocator_free(ma_slot_allocator* pAllocator, ma_uint64 ma_uint32 newBitfield; oldBitfield = ma_atomic_load_32(&pAllocator->pGroups[iGroup].bitfield); /* <-- This copy must happen. The compiler must not optimize this away. */ - newBitfield = oldBitfield & ~(1 << iBit); + newBitfield = oldBitfield & ~(1U << iBit); /* Debugging for checking for double-frees. */ #if defined(MA_DEBUG_OUTPUT) { - if ((oldBitfield & (1 << iBit)) == 0) { + if ((oldBitfield & (1U << iBit)) == 0) { MA_ASSERT(MA_FALSE); /* Double free detected.*/ } } @@ -20221,7 +20297,7 @@ static void ma_device__on_notification_stopped(ma_device* pDevice) } /* Not all platforms support reroute notifications. */ -#if !defined(MA_EMSCRIPTEN) +#if !defined(MA_EMSCRIPTEN) && !defined(MA_BSD) static void ma_device__on_notification_rerouted(ma_device* pDevice) { ma_device__on_notification(ma_device_notification_init(pDevice, ma_device_notification_type_rerouted)); @@ -20794,6 +20870,7 @@ static ma_result ma_device_audio_thread__default_read_write(ma_device* pDevice) for (;;) { ma_uint8 capturedClientData[MA_DATA_CONVERTER_STACK_BUFFER_SIZE]; ma_uint8 playbackClientData[MA_DATA_CONVERTER_STACK_BUFFER_SIZE]; + ma_uint64 playbackClientFramesProcessed = 0; ma_uint32 capturedClientDataCapInFrames = sizeof(capturedClientData) / ma_get_bytes_per_frame(pDevice->capture.format, pDevice->capture.channels); ma_uint32 playbackClientDataCapInFrames = sizeof(playbackClientData) / ma_get_bytes_per_frame(pDevice->playback.format, pDevice->playback.channels); ma_uint64 capturedClientFramesToProcessThisIteration = ma_min(capturedClientDataCapInFrames, playbackClientDataCapInFrames); @@ -20823,7 +20900,7 @@ static ma_result ma_device_audio_thread__default_read_write(ma_device* pDevice) for (;;) { ma_uint64 convertedClientFrameCount = capturedClientFramesToProcessThisIteration; ma_uint64 convertedDeviceFrameCount = playbackDeviceDataCapInFrames; - result = ma_data_converter_process_pcm_frames(&pDevice->playback.converter, playbackClientData, &convertedClientFrameCount, playbackDeviceData, &convertedDeviceFrameCount); + result = ma_data_converter_process_pcm_frames(&pDevice->playback.converter, ma_offset_pcm_frames_ptr(playbackClientData, playbackClientFramesProcessed, pDevice->playback.format, pDevice->playback.channels), &convertedClientFrameCount, playbackDeviceData, &convertedDeviceFrameCount); if (result != MA_SUCCESS) { break; } @@ -20834,7 +20911,8 @@ static ma_result ma_device_audio_thread__default_read_write(ma_device* pDevice) break; } - capturedClientFramesToProcessThisIteration -= (ma_uint32)convertedClientFrameCount; /* Safe cast. */ + playbackClientFramesProcessed += convertedClientFrameCount; + capturedClientFramesToProcessThisIteration -= convertedClientFrameCount; if (capturedClientFramesToProcessThisIteration == 0) { break; } @@ -30967,7 +31045,6 @@ static ma_result ma_result_from_pulse(int result) } } -#if 0 static ma_pa_sample_format_t ma_format_to_pulse(ma_format format) { if (ma_is_little_endian()) { @@ -30994,7 +31071,6 @@ static ma_pa_sample_format_t ma_format_to_pulse(ma_format format) default: return MA_PA_SAMPLE_INVALID; } } -#endif static ma_format ma_format_from_pulse(ma_pa_sample_format_t format) { @@ -31345,28 +31421,52 @@ static void ma_device_source_name_callback(ma_pa_context* pPulseContext, const m } #endif -static ma_result ma_context_get_sink_info__pulse(ma_context* pContext, const char* pDeviceName, ma_pa_sink_info* pSinkInfo) +static ma_result ma_context_get_sink_info__pulse(ma_context* pContext, ma_ptr pMainLoop, ma_ptr pPulseContext, const char* pDeviceName, ma_pa_sink_info* pSinkInfo) { ma_pa_operation* pOP; + ma_result result; + + MA_ZERO_OBJECT(pSinkInfo); - pOP = ((ma_pa_context_get_sink_info_by_name_proc)pContext->pulse.pa_context_get_sink_info_by_name)((ma_pa_context*)pContext->pulse.pPulseContext, pDeviceName, ma_device_sink_info_callback, pSinkInfo); + pOP = ((ma_pa_context_get_sink_info_by_name_proc)pContext->pulse.pa_context_get_sink_info_by_name)((ma_pa_context*)pPulseContext, pDeviceName, ma_device_sink_info_callback, pSinkInfo); if (pOP == NULL) { return MA_ERROR; } - return ma_wait_for_operation_and_unref__pulse(pContext, pContext->pulse.pMainLoop, pOP); + result = ma_wait_for_operation_and_unref__pulse(pContext, pMainLoop, pOP); + if (result != MA_SUCCESS) { + return result; + } + + if (pSinkInfo->sample_spec.channels == 0) { + return MA_NO_DEVICE; + } + + return MA_SUCCESS; } -static ma_result ma_context_get_source_info__pulse(ma_context* pContext, const char* pDeviceName, ma_pa_source_info* pSourceInfo) +static ma_result ma_context_get_source_info__pulse(ma_context* pContext, ma_ptr pMainLoop, ma_ptr pPulseContext, const char* pDeviceName, ma_pa_source_info* pSourceInfo) { ma_pa_operation* pOP; + ma_result result; - pOP = ((ma_pa_context_get_source_info_by_name_proc)pContext->pulse.pa_context_get_source_info_by_name)((ma_pa_context*)pContext->pulse.pPulseContext, pDeviceName, ma_device_source_info_callback, pSourceInfo); + MA_ZERO_OBJECT(pSourceInfo); + + pOP = ((ma_pa_context_get_source_info_by_name_proc)pContext->pulse.pa_context_get_source_info_by_name)((ma_pa_context*)pPulseContext, pDeviceName, ma_device_source_info_callback, pSourceInfo); if (pOP == NULL) { return MA_ERROR; } - return ma_wait_for_operation_and_unref__pulse(pContext, pContext->pulse.pMainLoop, pOP); + result = ma_wait_for_operation_and_unref__pulse(pContext, pMainLoop, pOP); + if (result != MA_SUCCESS) { + return result; + } + + if (pSourceInfo->sample_spec.channels == 0) { + return MA_NO_DEVICE; + } + + return MA_SUCCESS; } static ma_result ma_context_get_default_device_index__pulse(ma_context* pContext, ma_device_type deviceType, ma_uint32* pIndex) @@ -31382,7 +31482,7 @@ static ma_result ma_context_get_default_device_index__pulse(ma_context* pContext if (deviceType == ma_device_type_playback) { ma_pa_sink_info sinkInfo; - result = ma_context_get_sink_info__pulse(pContext, NULL, &sinkInfo); + result = ma_context_get_sink_info__pulse(pContext, pContext->pulse.pMainLoop, pContext->pulse.pPulseContext, NULL, &sinkInfo); if (result != MA_SUCCESS) { return result; } @@ -31394,7 +31494,7 @@ static ma_result ma_context_get_default_device_index__pulse(ma_context* pContext if (deviceType == ma_device_type_capture) { ma_pa_source_info sourceInfo; - result = ma_context_get_source_info__pulse(pContext, NULL, &sourceInfo); + result = ma_context_get_source_info__pulse(pContext, pContext->pulse.pMainLoop, pContext->pulse.pPulseContext, NULL, &sourceInfo); if (result != MA_SUCCESS) { return result; } @@ -31555,6 +31655,11 @@ static void ma_context_get_device_info_sink_callback__pulse(ma_pa_context* pPuls return; } + /* An error reply (e.g. the device no longer exists) passes a NULL pInfo. */ + if (pInfo == NULL) { + return; + } + MA_ASSERT(pData != NULL); pData->foundDevice = MA_TRUE; @@ -31592,6 +31697,11 @@ static void ma_context_get_device_info_source_callback__pulse(ma_pa_context* pPu return; } + /* An error reply (e.g. the device no longer exists) passes a NULL pInfo. */ + if (pInfo == NULL) { + return; + } + MA_ASSERT(pData != NULL); pData->foundDevice = MA_TRUE; @@ -31878,7 +31988,7 @@ static void ma_device_on_write__pulse(ma_pa_stream* pStream, size_t byteCount, v } result = ma_device_write_to_stream__pulse(pDevice, pStream, &framesProcessedThisIteration); - if (result != MA_SUCCESS) { + if (result != MA_SUCCESS || framesProcessedThisIteration == 0) { break; } @@ -32018,7 +32128,7 @@ static ma_result ma_device_init__pulse(ma_device* pDevice, const ma_device_confi } if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) { - result = ma_context_get_source_info__pulse(pDevice->pContext, devCapture, &sourceInfo); + result = ma_context_get_source_info__pulse(pDevice->pContext, pDevice->pulse.pMainLoop, pDevice->pulse.pPulseContext, devCapture, &sourceInfo); if (result != MA_SUCCESS) { ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, "[PulseAudio] Failed to retrieve source info for capture device."); goto on_error0; @@ -32027,6 +32137,9 @@ static ma_result ma_device_init__pulse(ma_device* pDevice, const ma_device_confi ss = sourceInfo.sample_spec; cmap = sourceInfo.channel_map; + /* Use the requested format if we have one. */ + ss.format = ma_format_to_pulse(pDescriptorCapture->format); + /* Use the requested channel count if we have one. */ if (pDescriptorCapture->channels != 0) { ss.channels = pDescriptorCapture->channels; @@ -32052,7 +32165,7 @@ static ma_result ma_device_init__pulse(ma_device* pDevice, const ma_device_confi } else { ss.format = MA_PA_SAMPLE_FLOAT32BE; } - streamFlags |= MA_PA_STREAM_FIX_FORMAT; + /*streamFlags |= MA_PA_STREAM_FIX_FORMAT;*/ /* Using PA_STREAM_FIX_FORMAT will make it so PulseAudio can possibly pick S24_32 which is not supported by miniaudio and will thus result in an error later on. By excluding it, PulseAudio should always pick F32. */ ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, "[PulseAudio] sample_spec.format not supported by miniaudio. Defaulting to PA_SAMPLE_FLOAT32.\n"); } if (ss.rate == 0) { @@ -32175,7 +32288,7 @@ static ma_result ma_device_init__pulse(ma_device* pDevice, const ma_device_confi } if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) { - result = ma_context_get_sink_info__pulse(pDevice->pContext, devPlayback, &sinkInfo); + result = ma_context_get_sink_info__pulse(pDevice->pContext, pDevice->pulse.pMainLoop, pDevice->pulse.pPulseContext, devPlayback, &sinkInfo); if (result != MA_SUCCESS) { ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, "[PulseAudio] Failed to retrieve sink info for playback device.\n"); goto on_error2; @@ -32184,6 +32297,9 @@ static ma_result ma_device_init__pulse(ma_device* pDevice, const ma_device_confi ss = sinkInfo.sample_spec; cmap = sinkInfo.channel_map; + /* Use the requested format if we have one. */ + ss.format = ma_format_to_pulse(pDescriptorPlayback->format); + /* Use the requested channel count if we have one. */ if (pDescriptorPlayback->channels != 0) { ss.channels = pDescriptorPlayback->channels; @@ -32210,7 +32326,7 @@ static ma_result ma_device_init__pulse(ma_device* pDevice, const ma_device_confi } else { ss.format = MA_PA_SAMPLE_FLOAT32BE; } - streamFlags |= MA_PA_STREAM_FIX_FORMAT; + /*streamFlags |= MA_PA_STREAM_FIX_FORMAT;*/ /* Using PA_STREAM_FIX_FORMAT will make it so PulseAudio can possibly pick S24_32 which is not supported by miniaudio and will thus result in an error later on. By excluding it, PulseAudio should always pick F32. */ ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, "[PulseAudio] sample_spec.format not supported by miniaudio. Defaulting to PA_SAMPLE_FLOAT32.\n"); } if (ss.rate == 0) { @@ -32375,6 +32491,16 @@ static ma_result ma_device_init__pulse(ma_device* pDevice, const ma_device_confi ((ma_pa_stream_unref_proc)pDevice->pContext->pulse.pa_stream_unref)((ma_pa_stream*)pDevice->pulse.pStreamCapture); } on_error0: + /* Release the device's context and main loop, in the same order as ma_device_uninit__pulse(). */ + if (pDevice->pulse.pPulseContext != NULL) { + ((ma_pa_context_disconnect_proc)pDevice->pContext->pulse.pa_context_disconnect)((ma_pa_context*)pDevice->pulse.pPulseContext); + ((ma_pa_context_unref_proc)pDevice->pContext->pulse.pa_context_unref)((ma_pa_context*)pDevice->pulse.pPulseContext); + pDevice->pulse.pPulseContext = NULL; + } + if (pDevice->pulse.pMainLoop != NULL) { + ((ma_pa_mainloop_free_proc)pDevice->pContext->pulse.pa_mainloop_free)((ma_pa_mainloop*)pDevice->pulse.pMainLoop); + pDevice->pulse.pMainLoop = NULL; + } return result; } @@ -32441,12 +32567,19 @@ static ma_result ma_device_start__pulse(ma_device* pDevice) } if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) { + ma_uint64 framesProcessed; + /* - We need to fill some data before uncorking. Not doing this will result in the write callback - never getting fired. We're not going to abort if writing fails because I still want the device - to get uncorked. + Fill all available blocks before uncorking. pa_stream_begin_write() can return less than the + writable size, leaving an outstanding request that may not trigger another write callback. + Always try to uncork, even if writing fails. */ - ma_device_write_to_stream__pulse(pDevice, (ma_pa_stream*)(pDevice->pulse.pStreamPlayback), NULL); /* No need to check the result here. Always want to fall through an uncork.*/ + for (;;) { + result = ma_device_write_to_stream__pulse(pDevice, (ma_pa_stream*)(pDevice->pulse.pStreamPlayback), &framesProcessed); + if (result != MA_SUCCESS || framesProcessed == 0) { + break; + } + } result = ma_device__cork_stream__pulse(pDevice, ma_device_type_playback, 0); if (result != MA_SUCCESS) { @@ -33810,7 +33943,7 @@ static ma_result ma_get_channel_map_from_AudioChannelLayout(AudioChannelLayout* UInt32 iBit; AudioChannelBitmap bitmap = pChannelLayout->mChannelBitmap; for (iBit = 0; iBit < 32 && iChannel < channelMapCap; ++iBit) { - AudioChannelBitmap bit = bitmap & (1 << iBit); + AudioChannelBitmap bit = bitmap & (1U << iBit); if (bit != 0) { pChannelMap[iChannel++] = ma_channel_from_AudioChannelBit(bit); } @@ -37994,10 +38127,6 @@ static ma_result ma_device_init_fd__audio4(ma_device* pDevice, const ma_device_c "/dev/audio", "/dev/audio0" }; - const char* pDefaultDeviceCtlNames[] = { - "/dev/audioctl", - "/dev/audioctl0" - }; int fd; int fdFlags = 0; size_t iDefaultDevice = (size_t)-1; @@ -38050,6 +38179,10 @@ static ma_result ma_device_init_fd__audio4(ma_device* pDevice, const ma_device_c #if !defined(MA_AUDIO4_USE_NEW_API) /* Old API */ { + const char* pDefaultDeviceCtlNames[] = { + "/dev/audioctl", + "/dev/audioctl0" + }; audio_info_t fdInfo; int fdInfoResult = -1; @@ -39315,9 +39448,17 @@ static void ma_stream_error_callback__aaudio(ma_AAudioStream* pStream, void* pUs When we get an error, we'll assume that the stream is in an erroneous state and needs to be restarted. From the documentation, we cannot do this from the error callback. Therefore we are going to use an event thread for the AAudio backend to do this cleanly and safely. + + The job is counted before isTearingDown is checked. ma_device_uninit__aaudio() sets isTearingDown and + then waits for the count to drain, so either it sees this job and waits for it, or we see the flag here + and post nothing. Without this a job still queued when the device is uninitialized would lock the + destroyed rerouteLock. */ + ma_atomic_fetch_add_32(&pDevice->aaudio.pendingReroutes, 1); + if (ma_atomic_bool32_get(&pDevice->aaudio.isTearingDown)) { ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, "[AAudio] Device Disconnected. Tearing down device.\n"); + ma_atomic_fetch_sub_32(&pDevice->aaudio.pendingReroutes, 1); } else { job = ma_job_init(MA_JOB_TYPE_DEVICE_AAUDIO_REROUTE); @@ -39332,6 +39473,7 @@ static void ma_stream_error_callback__aaudio(ma_AAudioStream* pStream, void* pUs result = ma_device_job_thread_post(&pDevice->pContext->aaudio.jobThread, &job); if (result != MA_SUCCESS) { ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, "[AAudio] Device Disconnected. Failed to post job for rerouting.\n"); + ma_atomic_fetch_sub_32(&pDevice->aaudio.pendingReroutes, 1); return; } } @@ -39673,6 +39815,15 @@ static ma_result ma_device_uninit__aaudio(ma_device* pDevice) */ ma_atomic_bool32_set(&pDevice->aaudio.isTearingDown, MA_TRUE); + /* + A reroute job can still be queued on the job thread, or be waiting for rerouteLock. It dereferences the + device and locks rerouteLock, so it must finish before the lock is destroyed below. With isTearingDown + set it returns without reinitializing anything, and no new job can be posted. + */ + while (ma_atomic_load_32(&pDevice->aaudio.pendingReroutes) > 0) { + ma_sleep(1); + } + /* Wait for any rerouting to finish before attempting to close the streams. */ ma_mutex_lock(&pDevice->aaudio.rerouteLock); { @@ -39743,11 +39894,13 @@ static ma_result ma_device_init_streams__aaudio(ma_device* pDevice, const ma_dev return MA_DEVICE_TYPE_NOT_SUPPORTED; } - pDevice->aaudio.usage = pConfig->aaudio.usage; - pDevice->aaudio.contentType = pConfig->aaudio.contentType; - pDevice->aaudio.inputPreset = pConfig->aaudio.inputPreset; - pDevice->aaudio.allowedCapturePolicy = pConfig->aaudio.allowedCapturePolicy; - pDevice->aaudio.noAutoStartAfterReroute = pConfig->aaudio.noAutoStartAfterReroute; + pDevice->aaudio.usage = pConfig->aaudio.usage; + pDevice->aaudio.contentType = pConfig->aaudio.contentType; + pDevice->aaudio.inputPreset = pConfig->aaudio.inputPreset; + pDevice->aaudio.allowedCapturePolicy = pConfig->aaudio.allowedCapturePolicy; + pDevice->aaudio.noAutoStartAfterReroute = pConfig->aaudio.noAutoStartAfterReroute; + pDevice->aaudio.enableCompatibilityWorkarounds = pConfig->aaudio.enableCompatibilityWorkarounds; + pDevice->aaudio.allowSetBufferCapacity = pConfig->aaudio.allowSetBufferCapacity; if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) { result = ma_device_init_by_type__aaudio(pDevice, pConfig, ma_device_type_capture, pDescriptorCapture, (ma_AAudioStream**)&pDevice->aaudio.pStreamCapture); @@ -39950,18 +40103,21 @@ static ma_result ma_device_reinit__aaudio(ma_device* pDevice, ma_device_type dev deviceConfig.capture.format = pDevice->capture.format; deviceConfig.capture.channels = pDevice->capture.channels; deviceConfig.sampleRate = pDevice->sampleRate; - deviceConfig.aaudio.usage = pDevice->aaudio.usage; - deviceConfig.aaudio.contentType = pDevice->aaudio.contentType; - deviceConfig.aaudio.inputPreset = pDevice->aaudio.inputPreset; - deviceConfig.aaudio.allowedCapturePolicy = pDevice->aaudio.allowedCapturePolicy; - deviceConfig.aaudio.noAutoStartAfterReroute = pDevice->aaudio.noAutoStartAfterReroute; - deviceConfig.periods = 1; + deviceConfig.aaudio.usage = pDevice->aaudio.usage; + deviceConfig.aaudio.contentType = pDevice->aaudio.contentType; + deviceConfig.aaudio.inputPreset = pDevice->aaudio.inputPreset; + deviceConfig.aaudio.allowedCapturePolicy = pDevice->aaudio.allowedCapturePolicy; + deviceConfig.aaudio.noAutoStartAfterReroute = pDevice->aaudio.noAutoStartAfterReroute; + deviceConfig.aaudio.enableCompatibilityWorkarounds = pDevice->aaudio.enableCompatibilityWorkarounds; + deviceConfig.aaudio.allowSetBufferCapacity = pDevice->aaudio.allowSetBufferCapacity; /* Try to get an accurate period size. */ if (deviceType == ma_device_type_playback || deviceType == ma_device_type_duplex) { deviceConfig.periodSizeInFrames = pDevice->playback.internalPeriodSizeInFrames; + deviceConfig.periods = pDevice->playback.internalPeriods; } else { deviceConfig.periodSizeInFrames = pDevice->capture.internalPeriodSizeInFrames; + deviceConfig.periods = pDevice->capture.internalPeriods; } if (deviceType == ma_device_type_capture || deviceType == ma_device_type_duplex || deviceType == ma_device_type_loopback) { @@ -40212,6 +40368,9 @@ static ma_result ma_job_process__device__aaudio_reroute(ma_job* pJob) } ma_mutex_unlock(&pDevice->aaudio.rerouteLock); + /* Must be the last access to pDevice: once the count reaches zero ma_device_uninit() may proceed. */ + ma_atomic_fetch_sub_32(&pDevice->aaudio.pendingReroutes, 1); + return result; } #else @@ -41234,7 +41393,7 @@ static ma_result ma_device_drain__opensl(ma_device* pDevice, ma_device_type devi MA_ASSERT(deviceType == ma_device_type_capture || deviceType == ma_device_type_playback); - if (pDevice->type == ma_device_type_capture) { + if (deviceType == ma_device_type_capture) { pBufferQueue = (SLAndroidSimpleBufferQueueItf)pDevice->opensl.pBufferQueueCapture; pDevice->opensl.isDrainingCapture = MA_TRUE; } else { @@ -41243,9 +41402,15 @@ static ma_result ma_device_drain__opensl(ma_device* pDevice, ma_device_type devi } for (;;) { + SLresult resultSL; SLAndroidSimpleBufferQueueState state; - MA_OPENSL_BUFFERQUEUE(pBufferQueue)->GetState(pBufferQueue, &state); + resultSL = MA_OPENSL_BUFFERQUEUE(pBufferQueue)->GetState(pBufferQueue, &state); + if (resultSL != SL_RESULT_SUCCESS) { + ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_WARNING, "[OpenSL] Failed to retrieve the state of the buffer queue while draining. Skipping drain."); + break; + } + if (state.count == 0) { break; } @@ -41253,7 +41418,7 @@ static ma_result ma_device_drain__opensl(ma_device* pDevice, ma_device_type devi ma_sleep(10); } - if (pDevice->type == ma_device_type_capture) { + if (deviceType == ma_device_type_capture) { pDevice->opensl.isDrainingCapture = MA_FALSE; } else { pDevice->opensl.isDrainingPlayback = MA_FALSE; @@ -41495,6 +41660,17 @@ Web Audio Backend #ifdef MA_HAS_WEBAUDIO #include +/* Emscripten's inline JavaScript syntax uses extensions to standard C and C++. */ +#if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdollar-in-identifier-extension" + #if __has_warning("-Wvariadic-macro-arguments-omitted") + #pragma clang diagnostic ignored "-Wvariadic-macro-arguments-omitted" + #else + #pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" + #endif +#endif + #ifndef MA_EMSCRIPTEN_MAJOR #if defined(__EMSCRIPTEN_MAJOR__) #define MA_EMSCRIPTEN_MAJOR __EMSCRIPTEN_MAJOR__ @@ -41541,7 +41717,7 @@ Web Audio Backend #define MA_WEBAUDIO_LATENCY_HINT_PLAYBACK "playback" #endif -static ma_bool32 ma_is_capture_supported__webaudio() +static ma_bool32 ma_is_capture_supported__webaudio(void) { return EM_ASM_INT({ return (navigator.mediaDevices !== undefined && navigator.mediaDevices.getUserMedia !== undefined); @@ -41670,7 +41846,6 @@ static ma_result ma_device_uninit__webaudio(ma_device* pDevice) emscripten_destroy_web_audio_node(pDevice->webaudio.audioWorklet); emscripten_destroy_audio_context(pDevice->webaudio.audioContext); - ma_free(pDevice->webaudio.pStackBuffer, &pDevice->pContext->allocationCallbacks); } #else { @@ -41982,7 +42157,11 @@ static void ma_audio_worklet_thread_initialized__webaudio(EMSCRIPTEN_WEBAUDIO_T MA_ASSERT(pParameters != NULL); if (success == EM_FALSE) { - pParameters->pDevice->webaudio.initResult = MA_ERROR; + ma_device* pDevice; + + pDevice = pParameters->pDevice; + ma_free(pParameters, &pDevice->pContext->allocationCallbacks); + pDevice->webaudio.initResult = MA_ERROR; return; } @@ -42015,6 +42194,7 @@ static ma_result ma_device_init__webaudio(ma_device* pDevice, const ma_device_co ma_audio_worklet_thread_initialized_data* pInitParameters; void* pStackBuffer; + MA_ZERO_OBJECT(&audioContextAttributes); if (pConfig->performanceProfile == ma_performance_profile_conservative) { audioContextAttributes.latencyHint = MA_WEBAUDIO_LATENCY_HINT_PLAYBACK; } else { @@ -42055,7 +42235,7 @@ static ma_result ma_device_init__webaudio(ma_device* pDevice, const ma_device_co /* Our thread initialization parameters need to be allocated on the heap so they don't go out of scope. */ pInitParameters = (ma_audio_worklet_thread_initialized_data*)ma_malloc(sizeof(*pInitParameters), &pDevice->pContext->allocationCallbacks); if (pInitParameters == NULL) { - ma_free(pStackBuffer, &pDevice->pContext->allocationCallbacks); + ma_aligned_free(pStackBuffer, &pDevice->pContext->allocationCallbacks); emscripten_destroy_audio_context(pDevice->webaudio.audioContext); return MA_OUT_OF_MEMORY; } @@ -42071,13 +42251,16 @@ static ma_result ma_device_init__webaudio(ma_device* pDevice, const ma_device_co */ pDevice->webaudio.initResult = MA_BUSY; { + /* + Web Audio teardown does not wait for all stack users to stop. Retain this allocation after + handing it to Emscripten, including when initialization fails. + */ emscripten_start_wasm_audio_worklet_thread_async(pDevice->webaudio.audioContext, pStackBuffer, MA_AUDIO_WORKLETS_THREAD_STACK_SIZE, ma_audio_worklet_thread_initialized__webaudio, pInitParameters); } while (pDevice->webaudio.initResult == MA_BUSY) { emscripten_sleep(1); } /* We must wait for initialization to complete. We're just spinning here. The emscripten_sleep() call is why we need to build with `-sASYNCIFY`. */ /* Initialization is now complete. Descriptors were updated when the worklet was initialized. */ if (pDevice->webaudio.initResult != MA_SUCCESS) { - ma_free(pStackBuffer, &pDevice->pContext->allocationCallbacks); emscripten_destroy_audio_context(pDevice->webaudio.audioContext); return pDevice->webaudio.initResult; } @@ -42089,7 +42272,7 @@ static ma_result ma_device_init__webaudio(ma_device* pDevice, const ma_device_co state: 1, /* 1 = ma_device_state_stopped */ pDevice: $1 }); - }, pDevice->webaudio.audioContext, pDevice); + }, pDevice->webaudio.audioContext, (ma_uintptr)pDevice); return MA_SUCCESS; } @@ -42169,8 +42352,8 @@ static ma_result ma_device_init__webaudio(ma_device* pDevice, const ma_device_co /* The node processing callback. */ device.scriptNode.onaudioprocess = function(e) { - if (device.intermediaryBufferView == null || device.intermediaryBufferView.length == 0) { - device.intermediaryBufferView = new Float32Array(HEAPF32.buffer, pIntermediaryBuffer, bufferSize * channels); + if (device.intermediaryBufferView == null || device.intermediaryBufferView.length == 0 || device.intermediaryBufferView.buffer !== HEAPF32.buffer) { + device.intermediaryBufferView = new Float32Array(HEAPF32.buffer, Number(pIntermediaryBuffer), bufferSize * channels); } /* Do the capture side first. */ @@ -42227,7 +42410,7 @@ static ma_result ma_device_init__webaudio(ma_device* pDevice, const ma_device_co device.pDevice = pDevice; return window.miniaudio.track_device(device); - }, pConfig->deviceType, channels, sampleRate, periodSizeInFrames, pDevice->webaudio.pIntermediaryBuffer, pDevice); + }, pConfig->deviceType, channels, sampleRate, periodSizeInFrames, (ma_uintptr)pDevice->webaudio.pIntermediaryBuffer, (ma_uintptr)pDevice); if (deviceIndex < 0) { return MA_FAILED_TO_OPEN_BACKEND_DEVICE; @@ -42447,6 +42630,9 @@ static ma_result ma_context_init__webaudio(ma_context* pContext, const ma_contex return MA_SUCCESS; } +#if defined(__clang__) + #pragma clang diagnostic pop +#endif #endif /* MA_HAS_WEBAUDIO */ @@ -47690,7 +47876,6 @@ static MA_INLINE void ma_lpf1_process_pcm_frame_f32(ma_lpf1* pLPF, float* pY, co ma_uint32 c; const ma_uint32 channels = pLPF->channels; const float a = pLPF->a.f32; - const float b = 1 - a; MA_ASSUME(channels > 0); for (c = 0; c < channels; c += 1) { @@ -47698,9 +47883,9 @@ static MA_INLINE void ma_lpf1_process_pcm_frame_f32(ma_lpf1* pLPF, float* pY, co float x = pX[c]; float y; - y = b*x + a*r1; + y = x + a*(r1 - x); - pY[c] = y; + pY[c] = y; pLPF->pR1[c].f32 = y; } } @@ -48562,8 +48747,7 @@ static MA_INLINE void ma_hpf1_process_pcm_frame_f32(ma_hpf1* pHPF, float* pY, co { ma_uint32 c; const ma_uint32 channels = pHPF->channels; - const float a = 1 - pHPF->a.f32; - const float b = 1 - a; + const float b = pHPF->a.f32; MA_ASSUME(channels > 0); for (c = 0; c < channels; c += 1) { @@ -48571,7 +48755,7 @@ static MA_INLINE void ma_hpf1_process_pcm_frame_f32(ma_hpf1* pHPF, float* pY, co float x = pX[c]; float y; - y = b*x - a*r1; + y = b*(x + r1) - r1; pY[c] = y; pHPF->pR1[c].f32 = y; @@ -50708,7 +50892,7 @@ static /*__attribute__((noinline))*/ ma_result ma_gainer_process_pcm_frames_inte if (pGainer->t >= pGainer->config.smoothTimeInFrames) { interpolatedFrameCount = 0; } else { - interpolatedFrameCount = pGainer->t - pGainer->config.smoothTimeInFrames; + interpolatedFrameCount = pGainer->config.smoothTimeInFrames - pGainer->t; if (interpolatedFrameCount > frameCount) { interpolatedFrameCount = frameCount; } @@ -50755,11 +50939,17 @@ static /*__attribute__((noinline))*/ ma_result ma_gainer_process_pcm_frames_inte __m128 runningGainDelta0 = _mm_set_ps(pRunningGainDelta[1], pRunningGainDelta[0], pRunningGainDelta[1], pRunningGainDelta[0]); __m128 runningGain0 = _mm_set_ps(pRunningGain[1] + pRunningGainDelta[1], pRunningGain[0] + pRunningGainDelta[0], pRunningGain[1], pRunningGain[0]); + /* Each iteration processes two frames. */ + runningGainDelta0 = _mm_add_ps(runningGainDelta0, runningGainDelta0); + for (; iFrame < unrolledLoopCount; iFrame += 1) { _mm_storeu_ps(&pFramesOutF32[iFrame*4 + 0], _mm_mul_ps(_mm_loadu_ps(&pFramesInF32[iFrame*4 + 0]), runningGain0)); runningGain0 = _mm_add_ps(runningGain0, runningGainDelta0); } + /* Keep the next frame's gains for the scalar remainder. */ + _mm_storeu_ps(pRunningGain, runningGain0); + iFrame = unrolledLoopCount << 1; } else #endif @@ -50786,10 +50976,10 @@ static /*__attribute__((noinline))*/ ma_result ma_gainer_process_pcm_frames_inte pFramesOutF32[iFrame*4 + 3] = pFramesInF32[iFrame*4 + 3] * pRunningGain[3]; /* Move the running gain forward towards the new gain. */ - pRunningGain[0] += pRunningGainDelta[0]; - pRunningGain[1] += pRunningGainDelta[1]; - pRunningGain[2] += pRunningGainDelta[2]; - pRunningGain[3] += pRunningGainDelta[3]; + pRunningGain[0] += pRunningGainDelta[0] * 2; + pRunningGain[1] += pRunningGainDelta[1] * 2; + pRunningGain[2] += pRunningGainDelta[2] * 2; + pRunningGain[3] += pRunningGainDelta[3] * 2; } iFrame = unrolledLoopCount << 1; @@ -50824,6 +51014,11 @@ static /*__attribute__((noinline))*/ ma_result ma_gainer_process_pcm_frames_inte __m128 runningGain1 = _mm_set_ps(pRunningGain[1] + pRunningGainDelta[1], pRunningGain[0] + pRunningGainDelta[0], pRunningGain[5], pRunningGain[4]); __m128 runningGain2 = _mm_set_ps(pRunningGain[5] + pRunningGainDelta[5], pRunningGain[4] + pRunningGainDelta[4], pRunningGain[3] + pRunningGainDelta[3], pRunningGain[2] + pRunningGainDelta[2]); + /* Each iteration processes two frames. */ + runningGainDelta0 = _mm_add_ps(runningGainDelta0, runningGainDelta0); + runningGainDelta1 = _mm_add_ps(runningGainDelta1, runningGainDelta1); + runningGainDelta2 = _mm_add_ps(runningGainDelta2, runningGainDelta2); + for (; iFrame < unrolledLoopCount; iFrame += 1) { _mm_storeu_ps(&pFramesOutF32[iFrame*12 + 0], _mm_mul_ps(_mm_loadu_ps(&pFramesInF32[iFrame*12 + 0]), runningGain0)); _mm_storeu_ps(&pFramesOutF32[iFrame*12 + 4], _mm_mul_ps(_mm_loadu_ps(&pFramesInF32[iFrame*12 + 4]), runningGain1)); @@ -50834,6 +51029,10 @@ static /*__attribute__((noinline))*/ ma_result ma_gainer_process_pcm_frames_inte runningGain2 = _mm_add_ps(runningGain2, runningGainDelta2); } + /* Keep the next frame's gains for the scalar remainder. */ + _mm_storeu_ps(&pRunningGain[0], runningGain0); + _mm_storeu_ps(&pRunningGain[4], runningGain1); + iFrame = unrolledLoopCount << 1; } else #endif @@ -50899,8 +51098,8 @@ static /*__attribute__((noinline))*/ ma_result ma_gainer_process_pcm_frames_inte } } - pFramesOut = ma_offset_ptr(pFramesOut, interpolatedFrameCount * sizeof(float)); - pFramesIn = ma_offset_ptr(pFramesIn, interpolatedFrameCount * sizeof(float)); + pFramesOut = ma_offset_ptr(pFramesOut, interpolatedFrameCount * pGainer->config.channels * sizeof(float)); + pFramesIn = ma_offset_ptr(pFramesIn, interpolatedFrameCount * pGainer->config.channels * sizeof(float)); } frameCount -= interpolatedFrameCount; @@ -57031,7 +57230,7 @@ MA_API ma_result ma_data_converter_get_input_channel_map(const ma_data_converter } if (pConverter->hasChannelConverter) { - ma_channel_converter_get_output_channel_map(&pConverter->channelConverter, pChannelMap, channelMapCap); + ma_channel_converter_get_input_channel_map(&pConverter->channelConverter, pChannelMap, channelMapCap); } else { ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, pConverter->channelsOut); } @@ -57046,7 +57245,7 @@ MA_API ma_result ma_data_converter_get_output_channel_map(const ma_data_converte } if (pConverter->hasChannelConverter) { - ma_channel_converter_get_input_channel_map(&pConverter->channelConverter, pChannelMap, channelMapCap); + ma_channel_converter_get_output_channel_map(&pConverter->channelConverter, pChannelMap, channelMapCap); } else { ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, pConverter->channelsIn); } @@ -58905,7 +59104,7 @@ MA_API ma_result ma_duplex_rb_init(ma_format captureFormat, ma_uint32 captureCha } /* Seek forward a bit so we have a bit of a buffer in case of desyncs. */ - ma_pcm_rb_seek_write((ma_pcm_rb*)pRB, captureInternalPeriodSizeInFrames * 2); + ma_pcm_rb_seek_write((ma_pcm_rb*)pRB, (ma_uint32)ma_calculate_frame_count_after_resampling(sampleRate, captureInternalSampleRate, captureInternalPeriodSizeInFrames * 2)); return MA_SUCCESS; } @@ -61944,7 +62143,7 @@ extern "C" { #define MA_DR_WAV_XSTRINGIFY(x) MA_DR_WAV_STRINGIFY(x) #define MA_DR_WAV_VERSION_MAJOR 0 #define MA_DR_WAV_VERSION_MINOR 14 -#define MA_DR_WAV_VERSION_REVISION 5 +#define MA_DR_WAV_VERSION_REVISION 6 #define MA_DR_WAV_VERSION_STRING MA_DR_WAV_XSTRINGIFY(MA_DR_WAV_VERSION_MAJOR) "." MA_DR_WAV_XSTRINGIFY(MA_DR_WAV_VERSION_MINOR) "." MA_DR_WAV_XSTRINGIFY(MA_DR_WAV_VERSION_REVISION) #include #define MA_DR_WAVE_FORMAT_PCM 0x1 @@ -61952,6 +62151,7 @@ extern "C" { #define MA_DR_WAVE_FORMAT_IEEE_FLOAT 0x3 #define MA_DR_WAVE_FORMAT_ALAW 0x6 #define MA_DR_WAVE_FORMAT_MULAW 0x7 +#define MA_DR_WAVE_FORMAT_DTS 0x8 #define MA_DR_WAVE_FORMAT_DVI_ADPCM 0x11 #define MA_DR_WAVE_FORMAT_EXTENSIBLE 0xFFFE #define MA_DR_WAV_SEQUENTIAL 0x00000001 @@ -62379,7 +62579,7 @@ extern "C" { #define MA_DR_FLAC_XSTRINGIFY(x) MA_DR_FLAC_STRINGIFY(x) #define MA_DR_FLAC_VERSION_MAJOR 0 #define MA_DR_FLAC_VERSION_MINOR 13 -#define MA_DR_FLAC_VERSION_REVISION 3 +#define MA_DR_FLAC_VERSION_REVISION 4 #define MA_DR_FLAC_VERSION_STRING MA_DR_FLAC_XSTRINGIFY(MA_DR_FLAC_VERSION_MAJOR) "." MA_DR_FLAC_XSTRINGIFY(MA_DR_FLAC_VERSION_MINOR) "." MA_DR_FLAC_XSTRINGIFY(MA_DR_FLAC_VERSION_REVISION) #include #if defined(_MSC_VER) && _MSC_VER >= 1700 @@ -62546,12 +62746,17 @@ typedef struct ma_dr_flac_cache_t crc16Cache; ma_uint32 crc16CacheIgnoredBytes; } ma_dr_flac_bs; +typedef union +{ + ma_int32* pS32; + ma_int64* pS64; +} ma_dr_flac_samples; typedef struct { ma_uint8 subframeType; ma_uint8 wastedBitsPerSample; ma_uint8 lpcOrder; - ma_int32* pSamplesS32; + ma_dr_flac_samples samples; } ma_dr_flac_subframe; typedef struct { @@ -62585,7 +62790,9 @@ typedef struct ma_uint64 currentPCMFrame; ma_uint64 firstFLACFramePosInBytes; ma_dr_flac__memory_stream memoryStream; - ma_int32* pDecodedSamples; + ma_dr_flac_samples decodedSamples; + void* pDecodedSamplesAllocation; + ma_uint16 decodedSamplesCapacityInPCMFrames; ma_dr_flac_seekpoint* pSeekpoints; void* _oggbs; ma_bool32 _noSeekTableSeek : 1; @@ -62679,7 +62886,7 @@ extern "C" { #define MA_DR_MP3_XSTRINGIFY(x) MA_DR_MP3_STRINGIFY(x) #define MA_DR_MP3_VERSION_MAJOR 0 #define MA_DR_MP3_VERSION_MINOR 7 -#define MA_DR_MP3_VERSION_REVISION 3 +#define MA_DR_MP3_VERSION_REVISION 4 #define MA_DR_MP3_VERSION_STRING MA_DR_MP3_XSTRINGIFY(MA_DR_MP3_VERSION_MAJOR) "." MA_DR_MP3_XSTRINGIFY(MA_DR_MP3_VERSION_MINOR) "." MA_DR_MP3_XSTRINGIFY(MA_DR_MP3_VERSION_REVISION) #include #define MA_DR_MP3_MAX_PCM_FRAMES_PER_MP3_FRAME 1152 @@ -62691,7 +62898,11 @@ MA_API const char* ma_dr_mp3_version_string(void); #define MA_DR_MP3_MAX_L3_FRAME_PAYLOAD_BYTES MA_DR_MP3_MAX_FREE_FORMAT_FRAME_SIZE typedef struct { - int frame_bytes, channels, sample_rate, layer, bitrate_kbps; + int frame_bytes; + int channels; + int sample_rate; + int layer; + int bitrate_kbps; } ma_dr_mp3dec_frame_info; typedef struct { @@ -62784,6 +62995,7 @@ typedef struct ma_uint32 seekPointCount; ma_uint32 delayInPCMFrames; ma_uint32 paddingInPCMFrames; + ma_uint32 mp3FramesSkipped; ma_uint64 totalPCMFrameCount; ma_bool32 isVBR; ma_bool32 isCBR; @@ -62853,6 +63065,10 @@ static ma_result ma_decoder_seek_bytes(ma_decoder* pDecoder, ma_int64 byteOffset { MA_ASSERT(pDecoder != NULL); + if (pDecoder->onSeek == NULL) { + return MA_NOT_IMPLEMENTED; + } + return pDecoder->onSeek(pDecoder, byteOffset, origin); } @@ -70911,21 +71127,26 @@ static ma_result ma_resource_manager_data_buffer_node_acquire(ma_resource_manage } done: - /* If we failed to initialize the data buffer we need to free it. */ - if (result != MA_SUCCESS) { - if (nodeAlreadyExists == MA_FALSE) { - ma_resource_manager_data_buffer_node_remove(pResourceManager, pDataBufferNode); - ma_free(pDataBufferNode, &pResourceManager->config.allocationCallbacks); + { + ma_bool32 isDataOwnedByResourceManager = pDataBufferNode->isDataOwnedByResourceManager; + + /* If we failed to initialize the data buffer we need to free it. */ + if (result != MA_SUCCESS) { + if (nodeAlreadyExists == MA_FALSE) { + ma_resource_manager_data_buffer_node_remove(pResourceManager, pDataBufferNode); + ma_free(pDataBufferNode, &pResourceManager->config.allocationCallbacks); + pDataBufferNode = NULL; + } } - } - /* - The init notification needs to be uninitialized. This will be used if the node does not already - exist, and we've specified ASYNC | WAIT_INIT. - */ - if (nodeAlreadyExists == MA_FALSE && pDataBufferNode->isDataOwnedByResourceManager && (flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC) != 0) { - if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) { - ma_resource_manager_inline_notification_uninit(&initNotification); + /* + The init notification needs to be uninitialized. This will be used if the node does not already + exist, and we've specified ASYNC | WAIT_INIT. + */ + if (nodeAlreadyExists == MA_FALSE && isDataOwnedByResourceManager && (flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC) != 0) { + if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) { + ma_resource_manager_inline_notification_uninit(&initNotification); + } } } @@ -71028,7 +71249,20 @@ static ma_result ma_resource_manager_data_buffer_node_unacquire(ma_resource_mana /* Threading is enabled. The job queue will deal with the rest of the cleanup from here. */ } } else { - /* The sound isn't loading so we can just free the node here. */ + /* + The node isn't loading, so it can be freed synchronously here. But first wait for any + in-flight job to fully retire. A decoded-paged node reports MA_SUCCESS as soon as it is + playable, while page_data_buffer_node jobs keep decoding the remaining pages and only + increment the execution pointer at their tail. Freeing the node the instant we observe a + non-BUSY result therefore races the tail of such a job, which then reads/writes freed + memory. Wait for the execution pointer to catch up to the execution counter so no job is + still referencing the node, exactly as the asynchronous free path relies on for ordering. + (Mirrors the data-buffer fix in ma_resource_manager_data_buffer_uninit.) + */ + while (ma_atomic_load_32(&pDataBufferNode->executionPointer) != ma_atomic_load_32(&pDataBufferNode->executionCounter)) { + ma_yield(); + } + ma_resource_manager_data_buffer_node_free(pResourceManager, pDataBufferNode); } } @@ -71239,6 +71473,13 @@ static ma_result ma_resource_manager_data_buffer_init_ex_internal(ma_resource_ma if (result == MA_BUSY) { result = MA_SUCCESS; } + + /* The load job signals initialization before its final access to the data buffer. */ + if (result != MA_SUCCESS) { + while (ma_atomic_load_32(&pDataBuffer->executionPointer) != ma_atomic_load_32(&pDataBuffer->executionCounter)) { + ma_yield(); + } + } } } @@ -71334,7 +71575,19 @@ MA_API ma_result ma_resource_manager_data_buffer_uninit(ma_resource_manager_data } if (ma_resource_manager_data_buffer_result(pDataBuffer) == MA_SUCCESS) { - /* The data buffer can be deleted synchronously. */ + /* + The data buffer can be deleted synchronously. Before doing so we must wait for any + in-flight load job to fully retire. The load job publishes its result code (which is what + makes us take this branch) a few instructions before it finishes referencing pDataBuffer + and increments the execution pointer. Tearing the buffer down the instant we observe + MA_SUCCESS therefore races the tail of the load job, which then reads and writes freed + memory. Wait for the execution pointer to catch up to the execution counter so that no job + is still referencing the buffer, exactly as the asynchronous path relies on for ordering. + */ + while (ma_atomic_load_32(&pDataBuffer->executionPointer) != ma_atomic_load_32(&pDataBuffer->executionCounter)) { + ma_yield(); + } + return ma_resource_manager_data_buffer_uninit_internal(pDataBuffer); } else { /* @@ -71369,6 +71622,11 @@ MA_API ma_result ma_resource_manager_data_buffer_uninit(ma_resource_manager_data } ma_resource_manager_inline_notification_wait_and_uninit(¬ification); + + /* The free job signals completion before its final access to the data buffer. */ + while (ma_atomic_load_32(&pDataBuffer->executionPointer) != ma_atomic_load_32(&pDataBuffer->executionCounter)) { + ma_yield(); + } } return result; @@ -71964,6 +72222,11 @@ MA_API ma_result ma_resource_manager_data_stream_init_ex(ma_resource_manager* pR if we're not waiting because it will most likely be in a busy state. */ if (pDataStream->result != MA_SUCCESS) { + /* Wait for the load job's final access before the caller can free the data stream. */ + while (ma_atomic_load_32(&pDataStream->executionPointer) != ma_atomic_load_32(&pDataStream->executionCounter)) { + ma_yield(); + } + return pDataStream->result; } @@ -74453,11 +74716,11 @@ static ma_result ma_node_translate_bus_counts(const ma_node_config* pConfig, ma_ /* Some special rules for passthrough nodes. */ if ((pConfig->vtable->flags & MA_NODE_FLAG_PASSTHROUGH) != 0) { - if ((pConfig->vtable->inputBusCount != 0 && pConfig->vtable->inputBusCount != 1) || pConfig->vtable->outputBusCount != 1) { + if ((inputBusCount != 0 && inputBusCount != 1) || outputBusCount != 1) { return MA_INVALID_ARGS; /* Passthrough nodes must have exactly 1 output bus and either 0 or 1 input bus. */ } - if (pConfig->pInputChannels[0] != pConfig->pOutputChannels[0]) { + if (inputBusCount > 0 && pConfig->pInputChannels[0] != pConfig->pOutputChannels[0]) { return MA_INVALID_ARGS; /* Passthrough nodes must have the same number of channels between input and output nodes. */ } } @@ -75183,7 +75446,7 @@ static ma_result ma_node_read_pcm_frames(ma_node* pNode, ma_uint32 outputBusInde ppFramesOut[0] = pFramesOut; ppFramesIn[0] = ppFramesOut[0]; - result = ma_node_input_bus_read_pcm_frames(pNodeBase, &pNodeBase->pInputBuses[0], ppFramesIn[0], frameCount, &totalFramesRead, globalTime); + result = ma_node_input_bus_read_pcm_frames(pNodeBase, &pNodeBase->pInputBuses[0], ppFramesIn[0], frameCount, &totalFramesRead, globalTime + timeOffsetBeg); if (result == MA_SUCCESS) { /* Even though it's a passthrough, we still need to fire the callback. */ frameCountIn = totalFramesRead; @@ -75271,7 +75534,7 @@ static ma_result ma_node_read_pcm_frames(ma_node* pNode, ma_uint32 outputBusInde ppFramesIn[iInputBus] = ma_node_get_cached_input_ptr(pNode, iInputBus); /* Once we've determined our destination pointer we can read. Note that we must inspect the number of frames read and fill any leftovers with silence for safety. */ - result = ma_node_input_bus_read_pcm_frames(pNodeBase, &pNodeBase->pInputBuses[iInputBus], ppFramesIn[iInputBus], framesToProcessIn, &framesRead, globalTime); + result = ma_node_input_bus_read_pcm_frames(pNodeBase, &pNodeBase->pInputBuses[iInputBus], ppFramesIn[iInputBus], framesToProcessIn, &framesRead, globalTime + timeOffsetBeg); if (result != MA_SUCCESS) { /* It doesn't really matter if we fail because we'll just fill with silence. */ framesRead = 0; /* Just for safety, but I don't think it's really needed. */ @@ -76638,15 +76901,6 @@ static ma_result ma_engine_node_set_volume(ma_engine_node* pEngineNode, float vo ma_atomic_float_set(&pEngineNode->volume, volume); - /* If we're not smoothing we should bypass the volume gainer entirely. */ - if (pEngineNode->volumeSmoothTimeInPCMFrames == 0) { - /* We should always have an active spatializer because it can be enabled and disabled dynamically. We can just use that for holding our volume. */ - ma_spatializer_set_master_volume(&pEngineNode->spatializer, volume); - } else { - /* We're using volume smoothing, so apply the master volume to the gainer. */ - ma_gainer_set_gain(&pEngineNode->volumeGainer, volume); - } - return MA_SUCCESS; } @@ -76681,6 +76935,7 @@ static void ma_engine_node_process_pcm_frames__general(ma_engine_node* pEngineNo ma_bool32 isSpatializationEnabled; ma_bool32 isPanningEnabled; ma_bool32 isVolumeSmoothingEnabled; + float volume; frameCountIn = *pFrameCountIn; frameCountOut = *pFrameCountOut; @@ -76717,6 +76972,18 @@ static void ma_engine_node_process_pcm_frames__general(ma_engine_node* pEngineNo isPanningEnabled = pEngineNode->panner.pan != 0 && channelsOut != 1; isVolumeSmoothingEnabled = pEngineNode->volumeSmoothTimeInPCMFrames > 0; + /* Apply volume changes on the mixing thread so gainer state is not shared with the setter. */ + volume = ma_atomic_float_get(&pEngineNode->volume); + if (volume != pEngineNode->oldVolume) { + if (isVolumeSmoothingEnabled) { + ma_gainer_set_gain(&pEngineNode->volumeGainer, volume); + } else { + ma_spatializer_set_master_volume(&pEngineNode->spatializer, volume); + } + + pEngineNode->oldVolume = volume; + } + /* Keep going while we've still got data available for processing. */ while (totalFramesProcessedOut < frameCountOut) { /* @@ -76830,9 +77097,6 @@ static void ma_engine_node_process_pcm_frames__general(ma_engine_node* pEngineNo ma_spatializer_process_pcm_frames(&pEngineNode->spatializer, &pEngineNode->pEngine->listeners[iListener], pRunningFramesOut, pWorkingBuffer, framesJustProcessedOut); } else { /* No spatialization, but we still need to do channel conversion and master volume. */ - float volume; - ma_engine_node_get_volume(pEngineNode, &volume); /* Should never fail. */ - if (channelsIn == channelsOut) { /* No channel conversion required. Just copy straight to the output buffer. */ if (isVolumeSmoothingEnabled) { @@ -76992,6 +77256,15 @@ static void ma_engine_node_process_pcm_frames__sound(ma_node* pNode, const float } framesJustRead = totalFramesConverted; + + /* + When the last iteration of the loop above returns MA_AT_END, we can be in a situation where `result` is MA_AT_END and + framesJustRead is >0. This is never valid in miniaudio so we need to normalize this (MA_AT_END should only be used + when the number of frames read is exactly 0, everywhere in miniaudio). + */ + if (result == MA_AT_END && framesJustRead > 0) { + result = MA_SUCCESS; + } } MA_ASSERT(framesJustRead <= pSound->processingCacheCap); @@ -77234,6 +77507,7 @@ MA_API ma_result ma_engine_node_init_preallocated(const ma_engine_node_config* p pEngineNode->volumeSmoothTimeInPCMFrames = pConfig->volumeSmoothTimeInPCMFrames; pEngineNode->monoExpansionMode = pConfig->monoExpansionMode; ma_atomic_float_set(&pEngineNode->volume, 1); + pEngineNode->oldVolume = 1; pEngineNode->pitch = 1; pEngineNode->oldPitch = 1; pEngineNode->oldDopplerPitch = 1; @@ -80181,7 +80455,16 @@ static MA_INLINE void ma_dr_wav__bswap_samples(void* pSamples, ma_uint64 sampleC } break; default: { - MA_DR_WAV_ASSERT(MA_FALSE); + ma_uint64 iSample; + for (iSample = 0; iSample < sampleCount; iSample += 1) { + ma_uint8* pSample = (ma_uint8*)pSamples + (iSample * bytesPerSample); + ma_uint32 iByte; + for (iByte = 0; iByte < bytesPerSample / 2; iByte += 1) { + ma_uint8 temp = pSample[iByte]; + pSample[iByte] = pSample[bytesPerSample - iByte - 1]; + pSample[bytesPerSample - iByte - 1] = temp; + } + } } break; } } @@ -80225,7 +80508,7 @@ MA_PRIVATE MA_INLINE ma_uint32 ma_dr_wav_bytes_to_u32_ex(const ma_uint8* data, m return ma_dr_wav_bytes_to_u32_le(data); } } -MA_PRIVATE ma_int64 ma_dr_wav_aiff_extented_to_s64(const ma_uint8* data) +MA_PRIVATE ma_int64 ma_dr_wav_aiff_extended_to_s64(const ma_uint8* data) { ma_uint32 exponent = ((ma_uint32)data[0] << 8) | data[1]; ma_uint64 hi = ((ma_uint64)data[2] << 24) | ((ma_uint64)data[3] << 16) | ((ma_uint64)data[4] << 8) | ((ma_uint64)data[5] << 0); @@ -80239,7 +80522,7 @@ MA_PRIVATE ma_int64 ma_dr_wav_aiff_extented_to_s64(const ma_uint8* data) return sign ? MA_DR_WAV_INT64_MIN : MA_DR_WAV_INT64_MAX; } exponent -= 16383; - if (exponent > 63) { + if (exponent >= 63) { return sign ? MA_DR_WAV_INT64_MIN : MA_DR_WAV_INT64_MAX; } else if (exponent < 1) { return 0; @@ -80337,6 +80620,14 @@ MA_PRIVATE unsigned int ma_dr_wav__chunk_padding_size_w64(ma_uint64 chunkSize) { return (unsigned int)(chunkSize % 8); } +MA_PRIVATE unsigned int ma_dr_wav_calculate_padding_size(ma_dr_wav_container container, ma_uint64 chunkSize) +{ + if (container == ma_dr_wav_container_riff || container == ma_dr_wav_container_rf64) { + return ma_dr_wav__chunk_padding_size_riff(chunkSize); + } else { + return ma_dr_wav__chunk_padding_size_w64(chunkSize); + } +} MA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__msadpcm(ma_dr_wav* pWav, ma_uint64 samplesToRead, ma_int16* pBufferOut); MA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__ima(ma_dr_wav* pWav, ma_uint64 samplesToRead, ma_int16* pBufferOut); MA_PRIVATE ma_bool32 ma_dr_wav_init_write__internal(ma_dr_wav* pWav, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount); @@ -80361,7 +80652,12 @@ MA_PRIVATE ma_result ma_dr_wav__read_chunk_header(ma_dr_wav_read_proc onRead, vo if (onRead(pUserData, sizeInBytes, 8) != 8) { return MA_INVALID_FILE; } - pHeaderOut->sizeInBytes = ma_dr_wav_bytes_to_u64(sizeInBytes) - 24; + pHeaderOut->sizeInBytes = ma_dr_wav_bytes_to_u64(sizeInBytes); + if (pHeaderOut->sizeInBytes >= 24) { + pHeaderOut->sizeInBytes -= 24; + } else { + return MA_INVALID_FILE; + } pHeaderOut->paddingSize = ma_dr_wav__chunk_padding_size_w64(pHeaderOut->sizeInBytes); *pRunningBytesReadOut += 24; } else { @@ -80486,16 +80782,23 @@ MA_PRIVATE ma_uint8* ma_dr_wav__metadata_get_memory(ma_dr_wav__metadata_parser* pParser->pDataCursor += size; return pResult; } -MA_PRIVATE void ma_dr_wav__metadata_request_extra_memory_for_stage_2(ma_dr_wav__metadata_parser* pParser, size_t bytes, size_t align) +MA_PRIVATE void ma_dr_wav__metadata_request_extra_memory_for_stage_2(ma_dr_wav__metadata_parser* pParser, ma_uint64 bytes, size_t align) { - size_t extra = bytes + (align ? (align - 1) : 0); - pParser->extraCapacity += extra; + pParser->extraCapacity += bytes; + if (align > 0) { + pParser->extraCapacity += align - 1; + } } MA_PRIVATE ma_result ma_dr_wav__metadata_alloc(ma_dr_wav__metadata_parser* pParser, ma_allocation_callbacks* pAllocationCallbacks) { if (pParser->extraCapacity != 0 || pParser->metadataCount != 0) { + size_t capacity; + capacity = ma_dr_wav__metadata_memory_capacity(pParser); + if (capacity == 0) { + return MA_OUT_OF_MEMORY; + } pAllocationCallbacks->onFree(pParser->pData, pAllocationCallbacks->pUserData); - pParser->pData = (ma_uint8*)pAllocationCallbacks->onMalloc(ma_dr_wav__metadata_memory_capacity(pParser), pAllocationCallbacks->pUserData); + pParser->pData = (ma_uint8*)pAllocationCallbacks->onMalloc(capacity, pAllocationCallbacks->pUserData); pParser->pDataCursor = pParser->pData; if (pParser->pData == NULL) { return MA_OUT_OF_MEMORY; @@ -80505,6 +80808,17 @@ MA_PRIVATE ma_result ma_dr_wav__metadata_alloc(ma_dr_wav__metadata_parser* pPars } return MA_SUCCESS; } +MA_PRIVATE ma_dr_wav_metadata* ma_dr_wav__metadata_get_next_object(ma_dr_wav__metadata_parser* pParser) +{ + ma_dr_wav_metadata* pMetadata; + MA_DR_WAV_ASSERT(pParser->stage == ma_dr_wav__metadata_parser_stage_read); + if (pParser->metadataCursor >= pParser->metadataCount) { + return NULL; + } + pMetadata = &pParser->pMetadata[pParser->metadataCursor]; + MA_DR_WAV_ZERO_OBJECT(pMetadata); + return pMetadata; +} MA_PRIVATE size_t ma_dr_wav__metadata_parser_read(ma_dr_wav__metadata_parser* pParser, void* pBufferOut, size_t bytesToRead, ma_uint64* pCursor) { if (pCursor != NULL) { @@ -80513,24 +80827,46 @@ MA_PRIVATE size_t ma_dr_wav__metadata_parser_read(ma_dr_wav__metadata_parser* pP return pParser->onRead(pParser->pReadSeekUserData, pBufferOut, bytesToRead); } } -MA_PRIVATE ma_uint64 ma_dr_wav__read_smpl_to_metadata_obj(ma_dr_wav__metadata_parser* pParser, const ma_dr_wav_chunk_header* pChunkHeader, ma_dr_wav_metadata* pMetadata) +MA_PRIVATE ma_bool32 ma_dr_wav__metadata_validate_smpl_chunk(const ma_dr_wav_chunk_header* pChunkHeader, ma_uint32 loopCount, ma_uint32 samplerSpecificDataSizeInBytes, ma_uint64* pTrailingDataSizeInBytes) +{ + ma_uint64 remainingDataSizeInBytes; + MA_DR_WAV_ASSERT(pChunkHeader != NULL); + if (pChunkHeader->sizeInBytes < MA_DR_WAV_SMPL_BYTES) { + return MA_FALSE; + } + remainingDataSizeInBytes = pChunkHeader->sizeInBytes - MA_DR_WAV_SMPL_BYTES; + if ((ma_uint64)loopCount > remainingDataSizeInBytes / MA_DR_WAV_SMPL_LOOP_BYTES) { + return MA_FALSE; + } + remainingDataSizeInBytes -= (ma_uint64)loopCount * MA_DR_WAV_SMPL_LOOP_BYTES; + if ((ma_uint64)samplerSpecificDataSizeInBytes > remainingDataSizeInBytes) { + return MA_FALSE; + } + if (pTrailingDataSizeInBytes != NULL) { + *pTrailingDataSizeInBytes = remainingDataSizeInBytes - samplerSpecificDataSizeInBytes; + } + return MA_TRUE; +} +MA_PRIVATE ma_uint64 ma_dr_wav__read_smpl_to_metadata_obj(ma_dr_wav__metadata_parser* pParser, const ma_dr_wav_chunk_header* pChunkHeader, ma_dr_wav_metadata* pMetadata, ma_result* pResult) { ma_uint8 smplHeaderData[MA_DR_WAV_SMPL_BYTES]; ma_uint64 totalBytesRead = 0; size_t bytesJustRead; + *pResult = MA_INVALID_DATA; if (pMetadata == NULL) { return 0; } bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, smplHeaderData, sizeof(smplHeaderData), &totalBytesRead); MA_DR_WAV_ASSERT(pParser->stage == ma_dr_wav__metadata_parser_stage_read); MA_DR_WAV_ASSERT(pChunkHeader != NULL); - if (pMetadata != NULL && bytesJustRead == sizeof(smplHeaderData)) { + if (bytesJustRead == sizeof(smplHeaderData)) { ma_uint32 iSampleLoop; ma_uint32 loopCount; - ma_uint32 calculatedLoopCount; + ma_uint32 samplerSpecificDataSizeInBytes; + ma_uint64 trailingDataSizeInBytes; loopCount = ma_dr_wav_bytes_to_u32(smplHeaderData + 28); - calculatedLoopCount = (pChunkHeader->sizeInBytes - MA_DR_WAV_SMPL_BYTES) / MA_DR_WAV_SMPL_LOOP_BYTES; - if (loopCount != calculatedLoopCount) { + samplerSpecificDataSizeInBytes = ma_dr_wav_bytes_to_u32(smplHeaderData + 32); + if (!ma_dr_wav__metadata_validate_smpl_chunk(pChunkHeader, loopCount, samplerSpecificDataSizeInBytes, &trailingDataSizeInBytes)) { return totalBytesRead; } pMetadata->type = ma_dr_wav_metadata_type_smpl; @@ -80541,40 +80877,47 @@ MA_PRIVATE ma_uint64 ma_dr_wav__read_smpl_to_metadata_obj(ma_dr_wav__metadata_pa pMetadata->data.smpl.midiPitchFraction = ma_dr_wav_bytes_to_u32(smplHeaderData + 16); pMetadata->data.smpl.smpteFormat = ma_dr_wav_bytes_to_u32(smplHeaderData + 20); pMetadata->data.smpl.smpteOffset = ma_dr_wav_bytes_to_u32(smplHeaderData + 24); - pMetadata->data.smpl.sampleLoopCount = ma_dr_wav_bytes_to_u32(smplHeaderData + 28); - pMetadata->data.smpl.samplerSpecificDataSizeInBytes = ma_dr_wav_bytes_to_u32(smplHeaderData + 32); - if (pMetadata->data.smpl.sampleLoopCount == calculatedLoopCount) { - pMetadata->data.smpl.pLoops = (ma_dr_wav_smpl_loop*)ma_dr_wav__metadata_get_memory(pParser, sizeof(ma_dr_wav_smpl_loop) * pMetadata->data.smpl.sampleLoopCount, MA_DR_WAV_METADATA_ALIGNMENT); - for (iSampleLoop = 0; iSampleLoop < pMetadata->data.smpl.sampleLoopCount; ++iSampleLoop) { - ma_uint8 smplLoopData[MA_DR_WAV_SMPL_LOOP_BYTES]; - bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, smplLoopData, sizeof(smplLoopData), &totalBytesRead); - if (bytesJustRead == sizeof(smplLoopData)) { - pMetadata->data.smpl.pLoops[iSampleLoop].cuePointId = ma_dr_wav_bytes_to_u32(smplLoopData + 0); - pMetadata->data.smpl.pLoops[iSampleLoop].type = ma_dr_wav_bytes_to_u32(smplLoopData + 4); - pMetadata->data.smpl.pLoops[iSampleLoop].firstSampleOffset = ma_dr_wav_bytes_to_u32(smplLoopData + 8); - pMetadata->data.smpl.pLoops[iSampleLoop].lastSampleOffset = ma_dr_wav_bytes_to_u32(smplLoopData + 12); - pMetadata->data.smpl.pLoops[iSampleLoop].sampleFraction = ma_dr_wav_bytes_to_u32(smplLoopData + 16); - pMetadata->data.smpl.pLoops[iSampleLoop].playCount = ma_dr_wav_bytes_to_u32(smplLoopData + 20); - } else { - break; - } + pMetadata->data.smpl.sampleLoopCount = loopCount; + pMetadata->data.smpl.samplerSpecificDataSizeInBytes = samplerSpecificDataSizeInBytes; + pMetadata->data.smpl.pLoops = (ma_dr_wav_smpl_loop*)ma_dr_wav__metadata_get_memory(pParser, sizeof(ma_dr_wav_smpl_loop) * pMetadata->data.smpl.sampleLoopCount, MA_DR_WAV_METADATA_ALIGNMENT); + for (iSampleLoop = 0; iSampleLoop < pMetadata->data.smpl.sampleLoopCount; ++iSampleLoop) { + ma_uint8 smplLoopData[MA_DR_WAV_SMPL_LOOP_BYTES]; + bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, smplLoopData, sizeof(smplLoopData), &totalBytesRead); + if (bytesJustRead == sizeof(smplLoopData)) { + pMetadata->data.smpl.pLoops[iSampleLoop].cuePointId = ma_dr_wav_bytes_to_u32(smplLoopData + 0); + pMetadata->data.smpl.pLoops[iSampleLoop].type = ma_dr_wav_bytes_to_u32(smplLoopData + 4); + pMetadata->data.smpl.pLoops[iSampleLoop].firstSampleOffset = ma_dr_wav_bytes_to_u32(smplLoopData + 8); + pMetadata->data.smpl.pLoops[iSampleLoop].lastSampleOffset = ma_dr_wav_bytes_to_u32(smplLoopData + 12); + pMetadata->data.smpl.pLoops[iSampleLoop].sampleFraction = ma_dr_wav_bytes_to_u32(smplLoopData + 16); + pMetadata->data.smpl.pLoops[iSampleLoop].playCount = ma_dr_wav_bytes_to_u32(smplLoopData + 20); + } else { + return totalBytesRead; } - if (pMetadata->data.smpl.samplerSpecificDataSizeInBytes > 0) { - pMetadata->data.smpl.pSamplerSpecificData = ma_dr_wav__metadata_get_memory(pParser, pMetadata->data.smpl.samplerSpecificDataSizeInBytes, 1); - MA_DR_WAV_ASSERT(pMetadata->data.smpl.pSamplerSpecificData != NULL); - ma_dr_wav__metadata_parser_read(pParser, pMetadata->data.smpl.pSamplerSpecificData, pMetadata->data.smpl.samplerSpecificDataSizeInBytes, &totalBytesRead); + } + if (pMetadata->data.smpl.samplerSpecificDataSizeInBytes > 0) { + pMetadata->data.smpl.pSamplerSpecificData = ma_dr_wav__metadata_get_memory(pParser, pMetadata->data.smpl.samplerSpecificDataSizeInBytes, 1); + MA_DR_WAV_ASSERT(pMetadata->data.smpl.pSamplerSpecificData != NULL); + bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, pMetadata->data.smpl.pSamplerSpecificData, pMetadata->data.smpl.samplerSpecificDataSizeInBytes, &totalBytesRead); + if (bytesJustRead != pMetadata->data.smpl.samplerSpecificDataSizeInBytes) { + return totalBytesRead; } - } else { - MA_DR_WAV_ZERO_OBJECT(&pMetadata->data.smpl); } + if (trailingDataSizeInBytes > 0) { + if (!ma_dr_wav__seek_forward(pParser->onSeek, trailingDataSizeInBytes, pParser->pReadSeekUserData)) { + return totalBytesRead; + } + totalBytesRead += trailingDataSizeInBytes; + } + *pResult = MA_SUCCESS; } return totalBytesRead; } -MA_PRIVATE ma_uint64 ma_dr_wav__read_cue_to_metadata_obj(ma_dr_wav__metadata_parser* pParser, const ma_dr_wav_chunk_header* pChunkHeader, ma_dr_wav_metadata* pMetadata) +MA_PRIVATE ma_uint64 ma_dr_wav__read_cue_to_metadata_obj(ma_dr_wav__metadata_parser* pParser, const ma_dr_wav_chunk_header* pChunkHeader, ma_dr_wav_metadata* pMetadata, ma_result* pResult) { ma_uint8 cueHeaderSectionData[MA_DR_WAV_CUE_BYTES]; ma_uint64 totalBytesRead = 0; size_t bytesJustRead; + *pResult = MA_INVALID_DATA; if (pMetadata == NULL) { return 0; } @@ -80602,10 +80945,11 @@ MA_PRIVATE ma_uint64 ma_dr_wav__read_cue_to_metadata_obj(ma_dr_wav__metadata_par pMetadata->data.cue.pCuePoints[iCuePoint].blockStart = ma_dr_wav_bytes_to_u32(cuePointData + 16); pMetadata->data.cue.pCuePoints[iCuePoint].sampleOffset = ma_dr_wav_bytes_to_u32(cuePointData + 20); } else { - break; + return totalBytesRead; } } } + *pResult = MA_SUCCESS; } } return totalBytesRead; @@ -80771,7 +81115,11 @@ MA_PRIVATE ma_result ma_dr_wav_buffer_reader_read_u32(ma_dr_wav_buffer_reader* p MA_PRIVATE ma_uint64 ma_dr_wav__read_bext_to_metadata_obj(ma_dr_wav__metadata_parser* pParser, ma_dr_wav_metadata* pMetadata, ma_uint64 chunkSize) { ma_uint8 bextData[MA_DR_WAV_BEXT_BYTES]; - size_t bytesRead = ma_dr_wav__metadata_parser_read(pParser, bextData, sizeof(bextData), NULL); + size_t bytesRead; + if (pMetadata == NULL) { + return 0; + } + bytesRead = ma_dr_wav__metadata_parser_read(pParser, bextData, sizeof(bextData), NULL); MA_DR_WAV_ASSERT(pParser->stage == ma_dr_wav__metadata_parser_stage_read); if (bytesRead == sizeof(bextData)) { ma_dr_wav_buffer_reader reader; @@ -80804,8 +81152,9 @@ MA_PRIVATE ma_uint64 ma_dr_wav__read_bext_to_metadata_obj(ma_dr_wav__metadata_pa if (extraBytes > 0) { pMetadata->data.bext.pCodingHistory = (char*)ma_dr_wav__metadata_get_memory(pParser, extraBytes + 1, 1); MA_DR_WAV_ASSERT(pMetadata->data.bext.pCodingHistory != NULL); - bytesRead += ma_dr_wav__metadata_parser_read(pParser, pMetadata->data.bext.pCodingHistory, extraBytes, NULL); - pMetadata->data.bext.codingHistorySize = (ma_uint32)ma_dr_wav__strlen(pMetadata->data.bext.pCodingHistory); + pMetadata->data.bext.codingHistorySize = (ma_uint32)ma_dr_wav__metadata_parser_read(pParser, pMetadata->data.bext.pCodingHistory, extraBytes, NULL); + pMetadata->data.bext.pCodingHistory[pMetadata->data.bext.codingHistorySize] = '\0'; + bytesRead += pMetadata->data.bext.codingHistorySize; } else { pMetadata->data.bext.pCodingHistory = NULL; pMetadata->data.bext.codingHistorySize = 0; @@ -80814,26 +81163,41 @@ MA_PRIVATE ma_uint64 ma_dr_wav__read_bext_to_metadata_obj(ma_dr_wav__metadata_pa } return bytesRead; } +MA_PRIVATE size_t ma_dr_wav__metadata_read_string(ma_dr_wav__metadata_parser* pParser, ma_uint32 stringSize, char** ppString, ma_uint32* pStringLength) +{ + size_t bytesRead; + ma_uint32 stringLength; + char* pString; + *ppString = NULL; + *pStringLength = 0; + if (stringSize == 0) { + return 0; + } + pString = (char*)ma_dr_wav__metadata_get_memory(pParser, (size_t)stringSize + 1, 1); + bytesRead = ma_dr_wav__metadata_parser_read(pParser, pString, stringSize, NULL); + pString[bytesRead] = '\0'; + stringLength = (ma_uint32)bytesRead; + if (bytesRead > 0 && pString[bytesRead - 1] == '\0') { + stringLength -= 1; + } + *ppString = pString; + *pStringLength = stringLength; + return bytesRead; +} MA_PRIVATE ma_uint64 ma_dr_wav__read_list_label_or_note_to_metadata_obj(ma_dr_wav__metadata_parser* pParser, ma_dr_wav_metadata* pMetadata, ma_uint64 chunkSize, ma_dr_wav_metadata_type type) { ma_uint8 cueIDBuffer[MA_DR_WAV_LIST_LABEL_OR_NOTE_BYTES]; ma_uint64 totalBytesRead = 0; - size_t bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, cueIDBuffer, sizeof(cueIDBuffer), &totalBytesRead); + size_t bytesJustRead; + if (pMetadata == NULL) { + return 0; + } + bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, cueIDBuffer, sizeof(cueIDBuffer), &totalBytesRead); MA_DR_WAV_ASSERT(pParser->stage == ma_dr_wav__metadata_parser_stage_read); if (bytesJustRead == sizeof(cueIDBuffer)) { - ma_uint32 sizeIncludingNullTerminator; pMetadata->type = type; pMetadata->data.labelOrNote.cuePointId = ma_dr_wav_bytes_to_u32(cueIDBuffer); - sizeIncludingNullTerminator = (ma_uint32)chunkSize - MA_DR_WAV_LIST_LABEL_OR_NOTE_BYTES; - if (sizeIncludingNullTerminator > 0) { - pMetadata->data.labelOrNote.stringLength = sizeIncludingNullTerminator - 1; - pMetadata->data.labelOrNote.pString = (char*)ma_dr_wav__metadata_get_memory(pParser, sizeIncludingNullTerminator, 1); - MA_DR_WAV_ASSERT(pMetadata->data.labelOrNote.pString != NULL); - ma_dr_wav__metadata_parser_read(pParser, pMetadata->data.labelOrNote.pString, sizeIncludingNullTerminator, &totalBytesRead); - } else { - pMetadata->data.labelOrNote.stringLength = 0; - pMetadata->data.labelOrNote.pString = NULL; - } + totalBytesRead += ma_dr_wav__metadata_read_string(pParser, ((ma_uint32)chunkSize - MA_DR_WAV_LIST_LABEL_OR_NOTE_BYTES), &pMetadata->data.labelOrNote.pString, &pMetadata->data.labelOrNote.stringLength); } return totalBytesRead; } @@ -80841,10 +81205,13 @@ MA_PRIVATE ma_uint64 ma_dr_wav__read_list_labelled_cue_region_to_metadata_obj(ma { ma_uint8 buffer[MA_DR_WAV_LIST_LABELLED_TEXT_BYTES]; ma_uint64 totalBytesRead = 0; - size_t bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, buffer, sizeof(buffer), &totalBytesRead); + size_t bytesJustRead; + if (pMetadata == NULL) { + return 0; + } + bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, buffer, sizeof(buffer), &totalBytesRead); MA_DR_WAV_ASSERT(pParser->stage == ma_dr_wav__metadata_parser_stage_read); if (bytesJustRead == sizeof(buffer)) { - ma_uint32 sizeIncludingNullTerminator; pMetadata->type = ma_dr_wav_metadata_type_list_labelled_cue_region; pMetadata->data.labelledCueRegion.cuePointId = ma_dr_wav_bytes_to_u32(buffer + 0); pMetadata->data.labelledCueRegion.sampleLength = ma_dr_wav_bytes_to_u32(buffer + 4); @@ -80856,41 +81223,25 @@ MA_PRIVATE ma_uint64 ma_dr_wav__read_list_labelled_cue_region_to_metadata_obj(ma pMetadata->data.labelledCueRegion.language = ma_dr_wav_bytes_to_u16(buffer + 14); pMetadata->data.labelledCueRegion.dialect = ma_dr_wav_bytes_to_u16(buffer + 16); pMetadata->data.labelledCueRegion.codePage = ma_dr_wav_bytes_to_u16(buffer + 18); - sizeIncludingNullTerminator = (ma_uint32)chunkSize - MA_DR_WAV_LIST_LABELLED_TEXT_BYTES; - if (sizeIncludingNullTerminator > 0) { - pMetadata->data.labelledCueRegion.stringLength = sizeIncludingNullTerminator - 1; - pMetadata->data.labelledCueRegion.pString = (char*)ma_dr_wav__metadata_get_memory(pParser, sizeIncludingNullTerminator, 1); - MA_DR_WAV_ASSERT(pMetadata->data.labelledCueRegion.pString != NULL); - ma_dr_wav__metadata_parser_read(pParser, pMetadata->data.labelledCueRegion.pString, sizeIncludingNullTerminator, &totalBytesRead); - } else { - pMetadata->data.labelledCueRegion.stringLength = 0; - pMetadata->data.labelledCueRegion.pString = NULL; - } + totalBytesRead += ma_dr_wav__metadata_read_string(pParser, ((ma_uint32)chunkSize - MA_DR_WAV_LIST_LABELLED_TEXT_BYTES), &pMetadata->data.labelledCueRegion.pString, &pMetadata->data.labelledCueRegion.stringLength); } return totalBytesRead; } MA_PRIVATE ma_uint64 ma_dr_wav__metadata_process_info_text_chunk(ma_dr_wav__metadata_parser* pParser, ma_uint64 chunkSize, ma_dr_wav_metadata_type type) { ma_uint64 bytesRead = 0; - ma_uint32 stringSizeWithNullTerminator = (ma_uint32)chunkSize; if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) { pParser->metadataCount += 1; - ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, stringSizeWithNullTerminator, 1); + ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, chunkSize + 1, 1); } else { - ma_dr_wav_metadata* pMetadata = &pParser->pMetadata[pParser->metadataCursor]; + ma_dr_wav_metadata* pMetadata; + pMetadata = ma_dr_wav__metadata_get_next_object(pParser); + if (pMetadata == NULL) { + return 0; + } pMetadata->type = type; - if (stringSizeWithNullTerminator > 0) { - pMetadata->data.infoText.stringLength = stringSizeWithNullTerminator - 1; - pMetadata->data.infoText.pString = (char*)ma_dr_wav__metadata_get_memory(pParser, stringSizeWithNullTerminator, 1); - MA_DR_WAV_ASSERT(pMetadata->data.infoText.pString != NULL); - bytesRead = ma_dr_wav__metadata_parser_read(pParser, pMetadata->data.infoText.pString, (size_t)stringSizeWithNullTerminator, NULL); - if (bytesRead == chunkSize) { - pParser->metadataCursor += 1; - } else { - } - } else { - pMetadata->data.infoText.stringLength = 0; - pMetadata->data.infoText.pString = NULL; + bytesRead = ma_dr_wav__metadata_read_string(pParser, (ma_uint32)chunkSize, &pMetadata->data.infoText.pString, &pMetadata->data.infoText.stringLength); + if (bytesRead == chunkSize) { pParser->metadataCursor += 1; } } @@ -80909,7 +81260,11 @@ MA_PRIVATE ma_uint64 ma_dr_wav__metadata_process_unknown_chunk(ma_dr_wav__metada pParser->metadataCount += 1; ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, (size_t)chunkSize, 1); } else { - ma_dr_wav_metadata* pMetadata = &pParser->pMetadata[pParser->metadataCursor]; + ma_dr_wav_metadata* pMetadata; + pMetadata = ma_dr_wav__metadata_get_next_object(pParser); + if (pMetadata == NULL) { + return 0; + } pMetadata->type = ma_dr_wav_metadata_type_unknown; pMetadata->data.unknown.chunkLocation = location; pMetadata->data.unknown.id[0] = pChunkId[0]; @@ -80947,22 +81302,21 @@ MA_PRIVATE ma_uint64 ma_dr_wav__metadata_process_chunk(ma_dr_wav__metadata_parse bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, buffer, sizeof(buffer), &bytesRead); if (bytesJustRead == sizeof(buffer)) { ma_uint32 loopCount = ma_dr_wav_bytes_to_u32(buffer); - ma_uint64 calculatedLoopCount; - calculatedLoopCount = (pChunkHeader->sizeInBytes - MA_DR_WAV_SMPL_BYTES) / MA_DR_WAV_SMPL_LOOP_BYTES; - if (calculatedLoopCount == loopCount) { - bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, buffer, sizeof(buffer), &bytesRead); - if (bytesJustRead == sizeof(buffer)) { - ma_uint32 samplerSpecificDataSizeInBytes = ma_dr_wav_bytes_to_u32(buffer); + bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, buffer, sizeof(buffer), &bytesRead); + if (bytesJustRead == sizeof(buffer)) { + ma_uint32 samplerSpecificDataSizeInBytes = ma_dr_wav_bytes_to_u32(buffer); + if (ma_dr_wav__metadata_validate_smpl_chunk(pChunkHeader, loopCount, samplerSpecificDataSizeInBytes, NULL)) { pParser->metadataCount += 1; ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, sizeof(ma_dr_wav_smpl_loop) * loopCount, MA_DR_WAV_METADATA_ALIGNMENT); ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, samplerSpecificDataSizeInBytes, 1); + } else { } - } else { } } } else { - bytesRead = ma_dr_wav__read_smpl_to_metadata_obj(pParser, pChunkHeader, &pParser->pMetadata[pParser->metadataCursor]); - if (bytesRead == pChunkHeader->sizeInBytes) { + ma_result result; + bytesRead = ma_dr_wav__read_smpl_to_metadata_obj(pParser, pChunkHeader, ma_dr_wav__metadata_get_next_object(pParser), &result); + if (result == MA_SUCCESS && bytesRead == pChunkHeader->sizeInBytes) { pParser->metadataCursor += 1; } else { } @@ -80974,7 +81328,7 @@ MA_PRIVATE ma_uint64 ma_dr_wav__metadata_process_chunk(ma_dr_wav__metadata_parse if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) { pParser->metadataCount += 1; } else { - bytesRead = ma_dr_wav__read_inst_to_metadata_obj(pParser, &pParser->pMetadata[pParser->metadataCursor]); + bytesRead = ma_dr_wav__read_inst_to_metadata_obj(pParser, ma_dr_wav__metadata_get_next_object(pParser)); if (bytesRead == pChunkHeader->sizeInBytes) { pParser->metadataCursor += 1; } else { @@ -80987,7 +81341,7 @@ MA_PRIVATE ma_uint64 ma_dr_wav__metadata_process_chunk(ma_dr_wav__metadata_parse if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) { pParser->metadataCount += 1; } else { - bytesRead = ma_dr_wav__read_acid_to_metadata_obj(pParser, &pParser->pMetadata[pParser->metadataCursor]); + bytesRead = ma_dr_wav__read_acid_to_metadata_obj(pParser, ma_dr_wav__metadata_get_next_object(pParser)); if (bytesRead == pChunkHeader->sizeInBytes) { pParser->metadataCursor += 1; } else { @@ -81003,8 +81357,9 @@ MA_PRIVATE ma_uint64 ma_dr_wav__metadata_process_chunk(ma_dr_wav__metadata_parse cueCount = (size_t)(pChunkHeader->sizeInBytes - MA_DR_WAV_CUE_BYTES) / MA_DR_WAV_CUE_POINT_BYTES; ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, sizeof(ma_dr_wav_cue_point) * cueCount, MA_DR_WAV_METADATA_ALIGNMENT); } else { - bytesRead = ma_dr_wav__read_cue_to_metadata_obj(pParser, pChunkHeader, &pParser->pMetadata[pParser->metadataCursor]); - if (bytesRead == pChunkHeader->sizeInBytes) { + ma_result result; + bytesRead = ma_dr_wav__read_cue_to_metadata_obj(pParser, pChunkHeader, ma_dr_wav__metadata_get_next_object(pParser), &result); + if (result == MA_SUCCESS && bytesRead == pChunkHeader->sizeInBytes) { pParser->metadataCursor += 1; } else { } @@ -81039,7 +81394,7 @@ MA_PRIVATE ma_uint64 ma_dr_wav__metadata_process_chunk(ma_dr_wav__metadata_parse ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, allocSizeNeeded, 1); pParser->metadataCount += 1; } else { - bytesRead = ma_dr_wav__read_bext_to_metadata_obj(pParser, &pParser->pMetadata[pParser->metadataCursor], pChunkHeader->sizeInBytes); + bytesRead = ma_dr_wav__read_bext_to_metadata_obj(pParser, ma_dr_wav__metadata_get_next_object(pParser), pChunkHeader->sizeInBytes); if (bytesRead == pChunkHeader->sizeInBytes) { pParser->metadataCursor += 1; } else { @@ -81072,12 +81427,11 @@ MA_PRIVATE ma_uint64 ma_dr_wav__metadata_process_chunk(ma_dr_wav__metadata_parse subchunkDataSize = ma_dr_wav_bytes_to_u32(subchunkSizeBuffer); if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_label, "labl") || ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_note, "note")) { if (subchunkDataSize >= MA_DR_WAV_LIST_LABEL_OR_NOTE_BYTES) { - ma_uint64 stringSizeWithNullTerm = subchunkDataSize - MA_DR_WAV_LIST_LABEL_OR_NOTE_BYTES; if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) { pParser->metadataCount += 1; - ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, (size_t)stringSizeWithNullTerm, 1); + ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, (subchunkDataSize - MA_DR_WAV_LIST_LABEL_OR_NOTE_BYTES) + 1, 1); } else { - subchunkBytesRead = ma_dr_wav__read_list_label_or_note_to_metadata_obj(pParser, &pParser->pMetadata[pParser->metadataCursor], subchunkDataSize, ma_dr_wav_fourcc_equal(subchunkId, "labl") ? ma_dr_wav_metadata_type_list_label : ma_dr_wav_metadata_type_list_note); + subchunkBytesRead = ma_dr_wav__read_list_label_or_note_to_metadata_obj(pParser, ma_dr_wav__metadata_get_next_object(pParser), subchunkDataSize, ma_dr_wav_fourcc_equal(subchunkId, "labl") ? ma_dr_wav_metadata_type_list_label : ma_dr_wav_metadata_type_list_note); if (subchunkBytesRead == subchunkDataSize) { pParser->metadataCursor += 1; } else { @@ -81087,12 +81441,11 @@ MA_PRIVATE ma_uint64 ma_dr_wav__metadata_process_chunk(ma_dr_wav__metadata_parse } } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_labelled_cue_region, "ltxt")) { if (subchunkDataSize >= MA_DR_WAV_LIST_LABELLED_TEXT_BYTES) { - ma_uint64 stringSizeWithNullTerminator = subchunkDataSize - MA_DR_WAV_LIST_LABELLED_TEXT_BYTES; if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) { pParser->metadataCount += 1; - ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, (size_t)stringSizeWithNullTerminator, 1); + ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, (subchunkDataSize - MA_DR_WAV_LIST_LABELLED_TEXT_BYTES) + 1, 1); } else { - subchunkBytesRead = ma_dr_wav__read_list_labelled_cue_region_to_metadata_obj(pParser, &pParser->pMetadata[pParser->metadataCursor], subchunkDataSize); + subchunkBytesRead = ma_dr_wav__read_list_labelled_cue_region_to_metadata_obj(pParser, ma_dr_wav__metadata_get_next_object(pParser), subchunkDataSize); if (subchunkBytesRead == subchunkDataSize) { pParser->metadataCursor += 1; } else { @@ -81211,8 +81564,19 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p ma_bool8 foundChunk_data = MA_FALSE; ma_bool8 isAIFCFormType = MA_FALSE; ma_uint64 aiffFrameCount = 0; + ma_int64 fileSize = 0; + ma_bool32 hasKnownFileSize = MA_FALSE; cursor = 0; sequential = (flags & MA_DR_WAV_SEQUENTIAL) != 0; + isProcessingMetadata = !sequential && ((flags & MA_DR_WAV_WITH_METADATA) != 0); + if (isProcessingMetadata && pWav->onTell != NULL && pWav->onSeek != NULL) { + if (pWav->onSeek(pWav->pUserData, 0, MA_DR_WAV_SEEK_END)) { + if (pWav->onTell(pWav->pUserData, &fileSize)) { + hasKnownFileSize = MA_TRUE; + } + pWav->onSeek(pWav->pUserData, 0, MA_DR_WAV_SEEK_SET); + } + } MA_DR_WAV_ZERO_OBJECT(&fmt); if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, riff, sizeof(riff), &cursor) != sizeof(riff)) { return MA_FALSE; @@ -81332,7 +81696,6 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p cursor += bytesRemainingInChunk; } metadataStartPos = cursor; - isProcessingMetadata = !sequential && ((flags & MA_DR_WAV_WITH_METADATA) != 0); if (pWav->container != ma_dr_wav_container_riff && pWav->container != ma_dr_wav_container_rf64) { isProcessingMetadata = MA_FALSE; } @@ -81362,6 +81725,9 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p if (((pWav->container == ma_dr_wav_container_riff || pWav->container == ma_dr_wav_container_rifx || pWav->container == ma_dr_wav_container_rf64) && ma_dr_wav_fourcc_equal(header.id.fourcc, "fmt ")) || ((pWav->container == ma_dr_wav_container_w64) && ma_dr_wav_guid_equal(header.id.guid, ma_dr_wavGUID_W64_FMT))) { ma_uint8 fmtData[16]; + if (header.sizeInBytes < sizeof(fmtData)) { + return MA_FALSE; + } foundChunk_fmt = MA_TRUE; if (pWav->onRead(pWav->pUserData, fmtData, sizeof(fmtData)) != sizeof(fmtData)) { return MA_FALSE; @@ -81380,6 +81746,7 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p if (header.sizeInBytes > 16) { ma_uint8 fmt_cbSize[2]; int bytesReadSoFar = 0; + ma_uint64 leftoverBytes; if (pWav->onRead(pWav->pUserData, fmt_cbSize, sizeof(fmt_cbSize)) != sizeof(fmt_cbSize)) { return MA_FALSE; } @@ -81408,10 +81775,14 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p cursor += fmt.extendedSize; bytesReadSoFar += fmt.extendedSize; } - if (pWav->onSeek(pWav->pUserData, (int)(header.sizeInBytes - bytesReadSoFar), MA_DR_WAV_SEEK_CUR) == MA_FALSE) { + leftoverBytes = header.sizeInBytes - bytesReadSoFar; + if (leftoverBytes > 0x7FFFFFFF) { return MA_FALSE; } - cursor += (header.sizeInBytes - bytesReadSoFar); + if (pWav->onSeek(pWav->pUserData, (int)leftoverBytes, MA_DR_WAV_SEEK_CUR) == MA_FALSE) { + return MA_FALSE; + } + cursor += leftoverBytes; } if (header.paddingSize > 0) { if (ma_dr_wav__seek_forward(pWav->onSeek, header.paddingSize, pWav->pUserData) == MA_FALSE) { @@ -81443,16 +81814,18 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p ((pWav->container == ma_dr_wav_container_w64) && ma_dr_wav_guid_equal(header.id.guid, ma_dr_wavGUID_W64_FACT))) { if (pWav->container == ma_dr_wav_container_riff || pWav->container == ma_dr_wav_container_rifx) { ma_uint8 sampleCount[4]; + if (chunkSize < 4) { + return MA_FALSE; + } if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, &sampleCount, 4, &cursor) != 4) { return MA_FALSE; } chunkSize -= 4; - if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ADPCM) { - sampleCountFromFactChunk = ma_dr_wav_bytes_to_u32_ex(sampleCount, pWav->container); - } else { - sampleCountFromFactChunk = 0; - } + sampleCountFromFactChunk = ma_dr_wav_bytes_to_u32_ex(sampleCount, pWav->container); } else if (pWav->container == ma_dr_wav_container_w64) { + if (chunkSize < 8) { + return MA_FALSE; + } if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, &sampleCountFromFactChunk, 8, &cursor) != 8) { return MA_FALSE; } @@ -81492,7 +81865,7 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p channels = ma_dr_wav_bytes_to_u16_ex (commData + 0, pWav->container); frameCount = ma_dr_wav_bytes_to_u32_ex (commData + 2, pWav->container); sampleSizeInBits = ma_dr_wav_bytes_to_u16_ex (commData + 6, pWav->container); - sampleRate = ma_dr_wav_aiff_extented_to_s64(commData + 8); + sampleRate = ma_dr_wav_aiff_extended_to_s64(commData + 8); if (sampleRate < 0 || sampleRate > 0xFFFFFFFF) { return MA_FALSE; } @@ -81530,19 +81903,15 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p fmt.formatTag = compressionFormat; fmt.channels = channels; fmt.sampleRate = (ma_uint32)sampleRate; - fmt.bitsPerSample = sampleSizeInBits; - fmt.blockAlign = (ma_uint16)(fmt.channels * fmt.bitsPerSample / 8); + fmt.bitsPerSample = (sampleSizeInBits + 7) & ~7; + fmt.blockAlign = (ma_uint16)((ma_uint32)fmt.channels * fmt.bitsPerSample / 8); fmt.avgBytesPerSec = fmt.blockAlign * fmt.sampleRate; - if (fmt.blockAlign == 0 && compressionFormat == MA_DR_WAVE_FORMAT_DVI_ADPCM) { - fmt.blockAlign = 34 * fmt.channels; - } if (compressionFormat == MA_DR_WAVE_FORMAT_ALAW || compressionFormat == MA_DR_WAVE_FORMAT_MULAW) { if (fmt.bitsPerSample > 8) { fmt.bitsPerSample = 8; fmt.blockAlign = fmt.channels; } } - fmt.bitsPerSample += (fmt.bitsPerSample & 7); if (isAIFCFormType) { if (ma_dr_wav__seek_forward(pWav->onSeek, (chunkSize - commDataBytesToRead), pWav->pUserData) == MA_FALSE) { return MA_FALSE; @@ -81587,6 +81956,9 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p } } if (isProcessingMetadata) { + if (hasKnownFileSize && header.sizeInBytes > (ma_uint64)fileSize) { + return MA_FALSE; + } ma_dr_wav__metadata_process_chunk(&metadataParser, &header, ma_dr_wav_metadata_type_all_including_unknown); if (ma_dr_wav__seek_from_start(pWav->onSeek, cursor, pWav->pUserData) == MA_FALSE) { break; @@ -81611,6 +81983,9 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p if (translatedFormatTag == MA_DR_WAVE_FORMAT_EXTENSIBLE) { translatedFormatTag = ma_dr_wav_bytes_to_u16_ex(fmt.subFormat + 0, pWav->container); } + if ((pWav->container == ma_dr_wav_container_riff || pWav->container == ma_dr_wav_container_rifx) && translatedFormatTag != MA_DR_WAVE_FORMAT_ADPCM) { + sampleCountFromFactChunk = 0; + } if (!sequential) { if (!ma_dr_wav__seek_from_start(pWav->onSeek, pWav->dataChunkDataPos, pWav->pUserData)) { return MA_FALSE; @@ -81634,13 +82009,16 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p break; } metadataBytesRead = ma_dr_wav__metadata_process_chunk(&metadataParser, &header, ma_dr_wav_metadata_type_all_including_unknown); + if (metadataParser.metadataCursor == metadataParser.metadataCount) { + break; + } if (ma_dr_wav__seek_forward(pWav->onSeek, (header.sizeInBytes + header.paddingSize) - metadataBytesRead, pWav->pUserData) == MA_FALSE) { ma_dr_wav_free(metadataParser.pMetadata, &pWav->allocationCallbacks); return MA_FALSE; } } pWav->pMetadata = metadataParser.pMetadata; - pWav->metadataCount = metadataParser.metadataCount; + pWav->metadataCount = (ma_uint32)metadataParser.metadataCursor; } if (pWav->onTell != NULL && pWav->onSeek != NULL) { if (pWav->onSeek(pWav->pUserData, 0, MA_DR_WAV_SEEK_END) == MA_TRUE) { @@ -81699,6 +82077,10 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p blockCount += 1; } totalBlockHeaderSizeInBytes = blockCount * (6*fmt.channels); + if (totalBlockHeaderSizeInBytes >= dataChunkSize) { + ma_dr_wav_free(pWav->pMetadata, &pWav->allocationCallbacks); + return MA_FALSE; + } pWav->totalPCMFrameCount = ((dataChunkSize - totalBlockHeaderSizeInBytes) * 2) / fmt.channels; } if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_DVI_ADPCM) { @@ -81708,6 +82090,10 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p blockCount += 1; } totalBlockHeaderSizeInBytes = blockCount * (4*fmt.channels); + if (totalBlockHeaderSizeInBytes >= dataChunkSize) { + ma_dr_wav_free(pWav->pMetadata, &pWav->allocationCallbacks); + return MA_FALSE; + } pWav->totalPCMFrameCount = ((dataChunkSize - totalBlockHeaderSizeInBytes) * 2) / fmt.channels; pWav->totalPCMFrameCount += blockCount; } @@ -81722,16 +82108,6 @@ MA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_p ma_dr_wav_free(pWav->pMetadata, &pWav->allocationCallbacks); return MA_FALSE; } -#ifdef MA_DR_WAV_LIBSNDFILE_COMPAT - if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ADPCM) { - ma_uint64 blockCount = dataChunkSize / fmt.blockAlign; - pWav->totalPCMFrameCount = (((blockCount * (fmt.blockAlign - (6*pWav->channels))) * 2)) / fmt.channels; - } - if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_DVI_ADPCM) { - ma_uint64 blockCount = dataChunkSize / fmt.blockAlign; - pWav->totalPCMFrameCount = (((blockCount * (fmt.blockAlign - (4*pWav->channels))) * 2) + (blockCount * pWav->channels)) / fmt.channels; - } -#endif return MA_TRUE; } MA_API ma_bool32 ma_dr_wav_init(ma_dr_wav* pWav, ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks) @@ -82696,6 +83072,49 @@ MA_API ma_bool32 ma_dr_wav_init_memory_write_sequential_pcm_frames(ma_dr_wav* pW } return ma_dr_wav_init_memory_write_sequential(pWav, ppData, pDataSize, pFormat, totalPCMFrameCount*pFormat->channels, pAllocationCallbacks); } +MA_PRIVATE ma_uint32 ma_dr_wav_write_padding(ma_dr_wav* pWav) +{ + ma_uint32 paddingSize = ma_dr_wav_calculate_padding_size(pWav->container, pWav->dataChunkDataSize); + if (paddingSize > 0) { + ma_uint64 paddingData = 0; + ma_dr_wav__write(pWav, &paddingData, paddingSize); + } + return paddingSize; +} +MA_PRIVATE void ma_dr_wav_write_chunk_sizes(ma_dr_wav* pWav) +{ + if (pWav->onSeek != NULL && !pWav->isSequentialWrite) { + if (pWav->container == ma_dr_wav_container_riff) { + if (pWav->onSeek(pWav->pUserData, 4, MA_DR_WAV_SEEK_SET)) { + ma_uint32 riffChunkSize = ma_dr_wav__riff_chunk_size_riff(pWav->dataChunkDataSize, pWav->pMetadata, pWav->metadataCount); + ma_dr_wav__write_u32ne_to_le(pWav, riffChunkSize); + } + if (pWav->onSeek(pWav->pUserData, (int)pWav->dataChunkDataPos - 4, MA_DR_WAV_SEEK_SET)) { + ma_uint32 dataChunkSize = ma_dr_wav__data_chunk_size_riff(pWav->dataChunkDataSize); + ma_dr_wav__write_u32ne_to_le(pWav, dataChunkSize); + } + } else if (pWav->container == ma_dr_wav_container_w64) { + if (pWav->onSeek(pWav->pUserData, 16, MA_DR_WAV_SEEK_SET)) { + ma_uint64 riffChunkSize = ma_dr_wav__riff_chunk_size_w64(pWav->dataChunkDataSize); + ma_dr_wav__write_u64ne_to_le(pWav, riffChunkSize); + } + if (pWav->onSeek(pWav->pUserData, (int)pWav->dataChunkDataPos - 8, MA_DR_WAV_SEEK_SET)) { + ma_uint64 dataChunkSize = ma_dr_wav__data_chunk_size_w64(pWav->dataChunkDataSize); + ma_dr_wav__write_u64ne_to_le(pWav, dataChunkSize); + } + } else if (pWav->container == ma_dr_wav_container_rf64) { + int ds64BodyPos = 12 + 8; + if (pWav->onSeek(pWav->pUserData, ds64BodyPos + 0, MA_DR_WAV_SEEK_SET)) { + ma_uint64 riffChunkSize = ma_dr_wav__riff_chunk_size_rf64(pWav->dataChunkDataSize, pWav->pMetadata, pWav->metadataCount); + ma_dr_wav__write_u64ne_to_le(pWav, riffChunkSize); + } + if (pWav->onSeek(pWav->pUserData, ds64BodyPos + 8, MA_DR_WAV_SEEK_SET)) { + ma_uint64 dataChunkSize = ma_dr_wav__data_chunk_size_rf64(pWav->dataChunkDataSize); + ma_dr_wav__write_u64ne_to_le(pWav, dataChunkSize); + } + } + } +} MA_API ma_result ma_dr_wav_uninit(ma_dr_wav* pWav) { ma_result result = MA_SUCCESS; @@ -82703,48 +83122,8 @@ MA_API ma_result ma_dr_wav_uninit(ma_dr_wav* pWav) return MA_INVALID_ARGS; } if (pWav->onWrite != NULL) { - ma_uint32 paddingSize = 0; - if (pWav->container == ma_dr_wav_container_riff || pWav->container == ma_dr_wav_container_rf64) { - paddingSize = ma_dr_wav__chunk_padding_size_riff(pWav->dataChunkDataSize); - } else { - paddingSize = ma_dr_wav__chunk_padding_size_w64(pWav->dataChunkDataSize); - } - if (paddingSize > 0) { - ma_uint64 paddingData = 0; - ma_dr_wav__write(pWav, &paddingData, paddingSize); - } - if (pWav->onSeek && !pWav->isSequentialWrite) { - if (pWav->container == ma_dr_wav_container_riff) { - if (pWav->onSeek(pWav->pUserData, 4, MA_DR_WAV_SEEK_SET)) { - ma_uint32 riffChunkSize = ma_dr_wav__riff_chunk_size_riff(pWav->dataChunkDataSize, pWav->pMetadata, pWav->metadataCount); - ma_dr_wav__write_u32ne_to_le(pWav, riffChunkSize); - } - if (pWav->onSeek(pWav->pUserData, (int)pWav->dataChunkDataPos - 4, MA_DR_WAV_SEEK_SET)) { - ma_uint32 dataChunkSize = ma_dr_wav__data_chunk_size_riff(pWav->dataChunkDataSize); - ma_dr_wav__write_u32ne_to_le(pWav, dataChunkSize); - } - } else if (pWav->container == ma_dr_wav_container_w64) { - if (pWav->onSeek(pWav->pUserData, 16, MA_DR_WAV_SEEK_SET)) { - ma_uint64 riffChunkSize = ma_dr_wav__riff_chunk_size_w64(pWav->dataChunkDataSize); - ma_dr_wav__write_u64ne_to_le(pWav, riffChunkSize); - } - if (pWav->onSeek(pWav->pUserData, (int)pWav->dataChunkDataPos - 8, MA_DR_WAV_SEEK_SET)) { - ma_uint64 dataChunkSize = ma_dr_wav__data_chunk_size_w64(pWav->dataChunkDataSize); - ma_dr_wav__write_u64ne_to_le(pWav, dataChunkSize); - } - } else if (pWav->container == ma_dr_wav_container_rf64) { - int ds64BodyPos = 12 + 8; - if (pWav->onSeek(pWav->pUserData, ds64BodyPos + 0, MA_DR_WAV_SEEK_SET)) { - ma_uint64 riffChunkSize = ma_dr_wav__riff_chunk_size_rf64(pWav->dataChunkDataSize, pWav->pMetadata, pWav->metadataCount); - ma_dr_wav__write_u64ne_to_le(pWav, riffChunkSize); - } - if (pWav->onSeek(pWav->pUserData, ds64BodyPos + 8, MA_DR_WAV_SEEK_SET)) { - ma_uint64 dataChunkSize = ma_dr_wav__data_chunk_size_rf64(pWav->dataChunkDataSize); - ma_dr_wav__write_u64ne_to_le(pWav, dataChunkSize); - } - } - } if (pWav->isSequentialWrite) { + ma_dr_wav_write_padding(pWav); if (pWav->dataChunkDataSize != pWav->dataChunkDataSizeTargetWrite) { result = MA_INVALID_FILE; } @@ -82804,8 +83183,8 @@ MA_API size_t ma_dr_wav_read_raw(ma_dr_wav* pWav, size_t bytesToRead, void* pBuf } } } - pWav->readCursorInPCMFrames += bytesRead / bytesPerFrame; pWav->bytesRemaining -= bytesRead; + pWav->readCursorInPCMFrames = (pWav->dataChunkDataSize - pWav->bytesRemaining) / bytesPerFrame; return bytesRead; } MA_API ma_uint64 ma_dr_wav_read_pcm_frames_le(ma_dr_wav* pWav, ma_uint64 framesToRead, void* pBufferOut) @@ -82868,7 +83247,7 @@ MA_API ma_uint64 ma_dr_wav_read_pcm_frames(ma_dr_wav* pWav, ma_uint64 framesToRe } post_process: { - if (pWav->container == ma_dr_wav_container_aiff && pWav->bitsPerSample == 8 && pWav->aiff.isUnsigned == MA_FALSE) { + if (pWav->container == ma_dr_wav_container_aiff && pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_PCM && pWav->bitsPerSample == 8 && pWav->aiff.isUnsigned == MA_FALSE) { if (pBufferOut != NULL) { ma_uint64 iSample; for (iSample = 0; iSample < framesRead * pWav->channels; iSample += 1) { @@ -82943,7 +83322,6 @@ MA_API ma_bool32 ma_dr_wav_seek_to_pcm_frame(ma_dr_wav* pWav, ma_uint64 targetFr } } } else { - ma_uint64 totalSizeInBytes; ma_uint64 currentBytePos; ma_uint64 targetBytePos; ma_uint64 offset; @@ -82952,9 +83330,9 @@ MA_API ma_bool32 ma_dr_wav_seek_to_pcm_frame(ma_dr_wav* pWav, ma_uint64 targetFr if (bytesPerFrame == 0) { return MA_FALSE; } - totalSizeInBytes = pWav->totalPCMFrameCount * bytesPerFrame; - currentBytePos = totalSizeInBytes - pWav->bytesRemaining; + currentBytePos = pWav->dataChunkDataSize - pWav->bytesRemaining; targetBytePos = targetFrameIndex * bytesPerFrame; + pWav->readCursorInPCMFrames = currentBytePos / bytesPerFrame; if (currentBytePos < targetBytePos) { offset = (targetBytePos - currentBytePos); } else { @@ -82968,8 +83346,8 @@ MA_API ma_bool32 ma_dr_wav_seek_to_pcm_frame(ma_dr_wav* pWav, ma_uint64 targetFr if (!pWav->onSeek(pWav->pUserData, offset32, MA_DR_WAV_SEEK_CUR)) { return MA_FALSE; } - pWav->readCursorInPCMFrames += offset32 / bytesPerFrame; pWav->bytesRemaining -= offset32; + pWav->readCursorInPCMFrames = (pWav->dataChunkDataSize - pWav->bytesRemaining) / bytesPerFrame; offset -= offset32; } } @@ -83007,6 +83385,14 @@ MA_API size_t ma_dr_wav_write_raw(ma_dr_wav* pWav, size_t bytesToWrite, const vo } bytesWritten = pWav->onWrite(pWav->pUserData, pData, bytesToWrite); pWav->dataChunkDataSize += bytesWritten; + if (!pWav->isSequentialWrite) { + ma_uint32 padding; + padding = ma_dr_wav_write_padding(pWav); + ma_dr_wav_write_chunk_sizes(pWav); + if (pWav->onSeek != NULL) { + pWav->onSeek(pWav->pUserData, -(int)padding, MA_DR_WAV_SEEK_END); + } + } return bytesWritten; } MA_API ma_uint64 ma_dr_wav_write_pcm_frames_le(ma_dr_wav* pWav, ma_uint64 framesToWrite, const void* pData) @@ -83088,6 +83474,16 @@ MA_API ma_uint64 ma_dr_wav_write_pcm_frames(ma_dr_wav* pWav, ma_uint64 framesToW return ma_dr_wav_write_pcm_frames_be(pWav, framesToWrite, pData); } } +MA_PRIVATE MA_INLINE ma_int32 ma_dr_wav__msadpcm_calc_sample(ma_int32 prevSample0, ma_int32 prevSample1, ma_int32 coeff1, ma_int32 coeff2, ma_int32 nibble, ma_int32 delta) +{ + ma_int32 prediction; + prediction = ((prevSample1 * coeff1) + (prevSample0 * coeff2)) >> 8; + if (delta > (0x7FFFFFFF - 98304) / 8) { + return (ma_int32)ma_dr_wav_clamp(prediction + ((ma_int64)nibble * delta), -32768, 32767); + } else { + return ma_dr_wav_clamp(prediction + (nibble * delta), -32768, 32767); + } +} MA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__msadpcm(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut) { ma_uint64 totalFramesRead = 0; @@ -83178,15 +83574,11 @@ MA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__msadpcm(ma_dr_wav* pWav, ma_ if (pWav->msadpcm.predictor[0] >= ma_dr_wav_countof(coeff1Table) || pWav->msadpcm.predictor[0] >= ma_dr_wav_countof(coeff2Table)) { return totalFramesRead; } - newSample0 = ((pWav->msadpcm.prevFrames[0][1] * coeff1Table[pWav->msadpcm.predictor[0]]) + (pWav->msadpcm.prevFrames[0][0] * coeff2Table[pWav->msadpcm.predictor[0]])) >> 8; - newSample0 += nibble0 * pWav->msadpcm.delta[0]; - newSample0 = ma_dr_wav_clamp(newSample0, -32768, 32767); + newSample0 = ma_dr_wav__msadpcm_calc_sample(pWav->msadpcm.prevFrames[0][0], pWav->msadpcm.prevFrames[0][1], coeff1Table[pWav->msadpcm.predictor[0]], coeff2Table[pWav->msadpcm.predictor[0]], nibble0, pWav->msadpcm.delta[0]); pWav->msadpcm.delta[0] = (ma_int32)ma_dr_wav_clamp(((ma_int64)adaptationTable[((nibbles & 0xF0) >> 4)] * pWav->msadpcm.delta[0]) >> 8, 16, 0x7FFFFFFF); pWav->msadpcm.prevFrames[0][0] = pWav->msadpcm.prevFrames[0][1]; pWav->msadpcm.prevFrames[0][1] = newSample0; - newSample1 = ((pWav->msadpcm.prevFrames[0][1] * coeff1Table[pWav->msadpcm.predictor[0]]) + (pWav->msadpcm.prevFrames[0][0] * coeff2Table[pWav->msadpcm.predictor[0]])) >> 8; - newSample1 += nibble1 * pWav->msadpcm.delta[0]; - newSample1 = ma_dr_wav_clamp(newSample1, -32768, 32767); + newSample1 = ma_dr_wav__msadpcm_calc_sample(pWav->msadpcm.prevFrames[0][0], pWav->msadpcm.prevFrames[0][1], coeff1Table[pWav->msadpcm.predictor[0]], coeff2Table[pWav->msadpcm.predictor[0]], nibble1, pWav->msadpcm.delta[0]); pWav->msadpcm.delta[0] = (ma_int32)ma_dr_wav_clamp(((ma_int64)adaptationTable[((nibbles & 0x0F) >> 0)] * pWav->msadpcm.delta[0]) >> 8, 16, 0x7FFFFFFF); pWav->msadpcm.prevFrames[0][0] = pWav->msadpcm.prevFrames[0][1]; pWav->msadpcm.prevFrames[0][1] = newSample1; @@ -83199,18 +83591,14 @@ MA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__msadpcm(ma_dr_wav* pWav, ma_ if (pWav->msadpcm.predictor[0] >= ma_dr_wav_countof(coeff1Table) || pWav->msadpcm.predictor[0] >= ma_dr_wav_countof(coeff2Table)) { return totalFramesRead; } - newSample0 = ((pWav->msadpcm.prevFrames[0][1] * coeff1Table[pWav->msadpcm.predictor[0]]) + (pWav->msadpcm.prevFrames[0][0] * coeff2Table[pWav->msadpcm.predictor[0]])) >> 8; - newSample0 += nibble0 * pWav->msadpcm.delta[0]; - newSample0 = ma_dr_wav_clamp(newSample0, -32768, 32767); + newSample0 = ma_dr_wav__msadpcm_calc_sample(pWav->msadpcm.prevFrames[0][0], pWav->msadpcm.prevFrames[0][1], coeff1Table[pWav->msadpcm.predictor[0]], coeff2Table[pWav->msadpcm.predictor[0]], nibble0, pWav->msadpcm.delta[0]); pWav->msadpcm.delta[0] = (ma_int32)ma_dr_wav_clamp(((ma_int64)adaptationTable[((nibbles & 0xF0) >> 4)] * pWav->msadpcm.delta[0]) >> 8, 16, 0x7FFFFFFF); pWav->msadpcm.prevFrames[0][0] = pWav->msadpcm.prevFrames[0][1]; pWav->msadpcm.prevFrames[0][1] = newSample0; if (pWav->msadpcm.predictor[1] >= ma_dr_wav_countof(coeff1Table) || pWav->msadpcm.predictor[1] >= ma_dr_wav_countof(coeff2Table)) { return totalFramesRead; } - newSample1 = ((pWav->msadpcm.prevFrames[1][1] * coeff1Table[pWav->msadpcm.predictor[1]]) + (pWav->msadpcm.prevFrames[1][0] * coeff2Table[pWav->msadpcm.predictor[1]])) >> 8; - newSample1 += nibble1 * pWav->msadpcm.delta[1]; - newSample1 = ma_dr_wav_clamp(newSample1, -32768, 32767); + newSample1 = ma_dr_wav__msadpcm_calc_sample(pWav->msadpcm.prevFrames[1][0], pWav->msadpcm.prevFrames[1][1], coeff1Table[pWav->msadpcm.predictor[1]], coeff2Table[pWav->msadpcm.predictor[1]], nibble1, pWav->msadpcm.delta[1]); pWav->msadpcm.delta[1] = (ma_int32)ma_dr_wav_clamp(((ma_int64)adaptationTable[((nibbles & 0x0F) >> 0)] * pWav->msadpcm.delta[1]) >> 8, 16, 0x7FFFFFFF); pWav->msadpcm.prevFrames[1][0] = pWav->msadpcm.prevFrames[1][1]; pWav->msadpcm.prevFrames[1][1] = newSample1; @@ -83422,6 +83810,9 @@ MA_PRIVATE void ma_dr_wav__pcm_to_s16(ma_int16* pOut, const ma_uint8* pIn, size_ sample |= (ma_uint64)(pIn[j]) << shift; shift += 8; } + if (!ma_dr_wav__is_little_endian()) { + sample = ma_dr_wav__bswap64(sample); + } pIn += j; *pOut++ = (ma_int16)((ma_int64)sample >> 48); } @@ -83547,16 +83938,6 @@ MA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__alaw(ma_dr_wav* pWav, ma_uin break; } ma_dr_wav_alaw_to_s16(pBufferOut, sampleData, (size_t)samplesRead); - #ifdef MA_DR_WAV_LIBSNDFILE_COMPAT - { - if (pWav->container == ma_dr_wav_container_aiff) { - ma_uint64 iSample; - for (iSample = 0; iSample < samplesRead; iSample += 1) { - pBufferOut[iSample] = -pBufferOut[iSample]; - } - } - } - #endif pBufferOut += samplesRead; framesToRead -= framesRead; totalFramesRead += framesRead; @@ -83595,29 +83976,31 @@ MA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__mulaw(ma_dr_wav* pWav, ma_ui break; } ma_dr_wav_mulaw_to_s16(pBufferOut, sampleData, (size_t)samplesRead); - #ifdef MA_DR_WAV_LIBSNDFILE_COMPAT - { - if (pWav->container == ma_dr_wav_container_aiff) { - ma_uint64 iSample; - for (iSample = 0; iSample < samplesRead; iSample += 1) { - pBufferOut[iSample] = -pBufferOut[iSample]; - } - } - } - #endif pBufferOut += samplesRead; framesToRead -= framesRead; totalFramesRead += framesRead; } return totalFramesRead; } +MA_PRIVATE ma_uint64 ma_dr_wav_seek_forward(ma_dr_wav* pWav, ma_uint64 framesToSeek) +{ + MA_DR_WAV_ASSERT(pWav != NULL); + MA_DR_WAV_ASSERT(framesToSeek > 0); + if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ADPCM) { + return ma_dr_wav_read_pcm_frames_s16__msadpcm(pWav, framesToSeek, NULL); + } + if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_DVI_ADPCM) { + return ma_dr_wav_read_pcm_frames_s16__ima(pWav, framesToSeek, NULL); + } + return ma_dr_wav_read_pcm_frames(pWav, framesToSeek, NULL); +} MA_API ma_uint64 ma_dr_wav_read_pcm_frames_s16(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut) { if (pWav == NULL || framesToRead == 0) { return 0; } if (pBufferOut == NULL) { - return ma_dr_wav_read_pcm_frames(pWav, framesToRead, NULL); + return ma_dr_wav_seek_forward(pWav, framesToRead); } if (framesToRead * pWav->channels * sizeof(ma_int16) > MA_SIZE_MAX) { framesToRead = MA_SIZE_MAX / sizeof(ma_int16) / pWav->channels; @@ -83691,30 +84074,34 @@ MA_API void ma_dr_wav_s32_to_s16(ma_int16* pOut, const ma_int32* pIn, size_t sam } MA_API void ma_dr_wav_f32_to_s16(ma_int16* pOut, const float* pIn, size_t sampleCount) { - int r; size_t i; for (i = 0; i < sampleCount; ++i) { float x = pIn[i]; - float c; - c = ((x < -1) ? -1 : ((x > 1) ? 1 : x)); - c = c + 1; - r = (int)(c * 32767.5f); - r = r - 32768; - pOut[i] = (short)r; + if (x != x) { + pOut[i] = 0; + } else if (x <= -1) { + pOut[i] = (-32767 - 1); + } else if (x >= 1) { + pOut[i] = 32767; + } else { + pOut[i] = (ma_int16)(x * 32768.0f); + } } } MA_API void ma_dr_wav_f64_to_s16(ma_int16* pOut, const double* pIn, size_t sampleCount) { - int r; size_t i; for (i = 0; i < sampleCount; ++i) { double x = pIn[i]; - double c; - c = ((x < -1) ? -1 : ((x > 1) ? 1 : x)); - c = c + 1; - r = (int)(c * 32767.5); - r = r - 32768; - pOut[i] = (short)r; + if (x != x) { + pOut[i] = 0; + } else if (x <= -1) { + pOut[i] = (-32767 - 1); + } else if (x >= 1) { + pOut[i] = 32767; + } else { + pOut[i] = (ma_int16)(x * 32768.0); + } } } MA_API void ma_dr_wav_alaw_to_s16(ma_int16* pOut, const ma_uint8* pIn, size_t sampleCount) @@ -83763,6 +84150,9 @@ MA_PRIVATE void ma_dr_wav__pcm_to_f32(float* pOut, const ma_uint8* pIn, size_t s sample |= (ma_uint64)(pIn[j]) << shift; shift += 8; } + if (!ma_dr_wav__is_little_endian()) { + sample = ma_dr_wav__bswap64(sample); + } pIn += j; *pOut++ = (float)((ma_int64)sample / 9223372036854775807.0); } @@ -83904,16 +84294,6 @@ MA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_f32__alaw(ma_dr_wav* pWav, ma_uin break; } ma_dr_wav_alaw_to_f32(pBufferOut, sampleData, (size_t)samplesRead); - #ifdef MA_DR_WAV_LIBSNDFILE_COMPAT - { - if (pWav->container == ma_dr_wav_container_aiff) { - ma_uint64 iSample; - for (iSample = 0; iSample < samplesRead; iSample += 1) { - pBufferOut[iSample] = -pBufferOut[iSample]; - } - } - } - #endif pBufferOut += samplesRead; framesToRead -= framesRead; totalFramesRead += framesRead; @@ -83949,16 +84329,6 @@ MA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_f32__mulaw(ma_dr_wav* pWav, ma_ui break; } ma_dr_wav_mulaw_to_f32(pBufferOut, sampleData, (size_t)samplesRead); - #ifdef MA_DR_WAV_LIBSNDFILE_COMPAT - { - if (pWav->container == ma_dr_wav_container_aiff) { - ma_uint64 iSample; - for (iSample = 0; iSample < samplesRead; iSample += 1) { - pBufferOut[iSample] = -pBufferOut[iSample]; - } - } - } - #endif pBufferOut += samplesRead; framesToRead -= framesRead; totalFramesRead += framesRead; @@ -83971,7 +84341,7 @@ MA_API ma_uint64 ma_dr_wav_read_pcm_frames_f32(ma_dr_wav* pWav, ma_uint64 frames return 0; } if (pBufferOut == NULL) { - return ma_dr_wav_read_pcm_frames(pWav, framesToRead, NULL); + return ma_dr_wav_seek_forward(pWav, framesToRead); } if (framesToRead * pWav->channels * sizeof(float) > MA_SIZE_MAX) { framesToRead = MA_SIZE_MAX / sizeof(float) / pWav->channels; @@ -84015,18 +84385,9 @@ MA_API void ma_dr_wav_u8_to_f32(float* pOut, const ma_uint8* pIn, size_t sampleC if (pOut == NULL || pIn == NULL) { return; } -#ifdef MA_DR_WAV_LIBSNDFILE_COMPAT for (i = 0; i < sampleCount; ++i) { - *pOut++ = (pIn[i] / 256.0f) * 2 - 1; + *pOut++ = ((int)pIn[i] - 128) * 0.0078125f; } -#else - for (i = 0; i < sampleCount; ++i) { - float x = pIn[i]; - x = x * 0.00784313725490196078f; - x = x - 1; - *pOut++ = x; - } -#endif } MA_API void ma_dr_wav_s16_to_f32(float* pOut, const ma_int16* pIn, size_t sampleCount) { @@ -84127,6 +84488,9 @@ MA_PRIVATE void ma_dr_wav__pcm_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_ sample |= (ma_uint64)(pIn[j]) << shift; shift += 8; } + if (!ma_dr_wav__is_little_endian()) { + sample = ma_dr_wav__bswap64(sample); + } pIn += j; *pOut++ = (ma_int32)((ma_int64)sample >> 32); } @@ -84264,16 +84628,6 @@ MA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s32__alaw(ma_dr_wav* pWav, ma_uin break; } ma_dr_wav_alaw_to_s32(pBufferOut, sampleData, (size_t)samplesRead); - #ifdef MA_DR_WAV_LIBSNDFILE_COMPAT - { - if (pWav->container == ma_dr_wav_container_aiff) { - ma_uint64 iSample; - for (iSample = 0; iSample < samplesRead; iSample += 1) { - pBufferOut[iSample] = -pBufferOut[iSample]; - } - } - } - #endif pBufferOut += samplesRead; framesToRead -= framesRead; totalFramesRead += framesRead; @@ -84309,16 +84663,6 @@ MA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s32__mulaw(ma_dr_wav* pWav, ma_ui break; } ma_dr_wav_mulaw_to_s32(pBufferOut, sampleData, (size_t)samplesRead); - #ifdef MA_DR_WAV_LIBSNDFILE_COMPAT - { - if (pWav->container == ma_dr_wav_container_aiff) { - ma_uint64 iSample; - for (iSample = 0; iSample < samplesRead; iSample += 1) { - pBufferOut[iSample] = -pBufferOut[iSample]; - } - } - } - #endif pBufferOut += samplesRead; framesToRead -= framesRead; totalFramesRead += framesRead; @@ -84331,7 +84675,7 @@ MA_API ma_uint64 ma_dr_wav_read_pcm_frames_s32(ma_dr_wav* pWav, ma_uint64 frames return 0; } if (pBufferOut == NULL) { - return ma_dr_wav_read_pcm_frames(pWav, framesToRead, NULL); + return ma_dr_wav_seek_forward(pWav, framesToRead); } if (framesToRead * pWav->channels * sizeof(ma_int32) > MA_SIZE_MAX) { framesToRead = MA_SIZE_MAX / sizeof(ma_int32) / pWav->channels; @@ -84376,7 +84720,7 @@ MA_API void ma_dr_wav_u8_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t samp return; } for (i = 0; i < sampleCount; ++i) { - *pOut++ = ((int)pIn[i] - 128) << 24; + *pOut++ = ((int)pIn[i] - 128) * 16777216; } } MA_API void ma_dr_wav_s16_to_s32(ma_int32* pOut, const ma_int16* pIn, size_t sampleCount) @@ -84386,7 +84730,7 @@ MA_API void ma_dr_wav_s16_to_s32(ma_int32* pOut, const ma_int16* pIn, size_t sam return; } for (i = 0; i < sampleCount; ++i) { - *pOut++ = pIn[i] << 16; + *pOut++ = (ma_int32)pIn[i] * 65536; } } MA_API void ma_dr_wav_s24_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t sampleCount) @@ -84406,21 +84750,33 @@ MA_API void ma_dr_wav_s24_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t sam MA_API void ma_dr_wav_f32_to_s32(ma_int32* pOut, const float* pIn, size_t sampleCount) { size_t i; - if (pOut == NULL || pIn == NULL) { - return; - } for (i = 0; i < sampleCount; ++i) { - *pOut++ = (ma_int32)(2147483648.0f * pIn[i]); + float x = pIn[i]; + if (x != x) { + pOut[i] = 0; + } else if (x <= -1) { + pOut[i] = (-2147483647 - 1); + } else if (x >= 1) { + pOut[i] = 2147483647; + } else { + pOut[i] = (ma_int32)(x * 2147483648.0f); + } } } MA_API void ma_dr_wav_f64_to_s32(ma_int32* pOut, const double* pIn, size_t sampleCount) { size_t i; - if (pOut == NULL || pIn == NULL) { - return; - } for (i = 0; i < sampleCount; ++i) { - *pOut++ = (ma_int32)(2147483648.0 * pIn[i]); + double x = pIn[i]; + if (x != x) { + pOut[i] = 0; + } else if (x <= -1) { + pOut[i] = (-2147483647 - 1); + } else if (x >= 1) { + pOut[i] = 2147483647; + } else { + pOut[i] = (ma_int32)(x * 2147483648.0); + } } } MA_API void ma_dr_wav_alaw_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t sampleCount) @@ -84430,7 +84786,7 @@ MA_API void ma_dr_wav_alaw_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t sa return; } for (i = 0; i < sampleCount; ++i) { - *pOut++ = ((ma_int32)ma_dr_wav__alaw_to_s16(pIn[i])) << 16; + *pOut++ = (ma_int32)ma_dr_wav__alaw_to_s16(pIn[i]) * 65536; } } MA_API void ma_dr_wav_mulaw_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t sampleCount) @@ -84440,7 +84796,7 @@ MA_API void ma_dr_wav_mulaw_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t s return; } for (i= 0; i < sampleCount; ++i) { - *pOut++ = ((ma_int32)ma_dr_wav__mulaw_to_s16(pIn[i])) << 16; + *pOut++ = (ma_int32)ma_dr_wav__mulaw_to_s16(pIn[i]) * 65536; } } MA_PRIVATE ma_int16* ma_dr_wav__read_pcm_frames_and_close_s16(ma_dr_wav* pWav, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalFrameCount) @@ -85061,6 +85417,7 @@ static MA_INLINE ma_bool32 ma_dr_flac_has_sse41(void) #ifndef MA_DR_FLAC_ZERO_OBJECT #define MA_DR_FLAC_ZERO_OBJECT(p) MA_DR_FLAC_ZERO_MEMORY((p), sizeof(*(p))) #endif +#define MA_DR_FLAC_MIN(a, b) (((a) < (b)) ? (a) : (b)) #define MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE 64 #define MA_DR_FLAC_SUBFRAME_CONSTANT 0 #define MA_DR_FLAC_SUBFRAME_VERBATIM 1 @@ -85069,14 +85426,15 @@ static MA_INLINE ma_bool32 ma_dr_flac_has_sse41(void) #define MA_DR_FLAC_SUBFRAME_RESERVED 255 #define MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE 0 #define MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2 1 -#define MA_DR_FLAC_CHANNEL_ASSIGNMENT_INDEPENDENT 0 +#define MA_DR_FLAC_CHANNEL_ASSIGNMENT_INDEPENDENT_STEREO 1 #define MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE 8 #define MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE 9 #define MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE 10 +#define ma_dr_flac_align(x, a) ((((x) + (a) - 1) / (a)) * (a)) #define MA_DR_FLAC_SEEKPOINT_SIZE_IN_BYTES 18 #define MA_DR_FLAC_CUESHEET_TRACK_SIZE_IN_BYTES 36 #define MA_DR_FLAC_CUESHEET_TRACK_INDEX_SIZE_IN_BYTES 12 -#define ma_dr_flac_align(x, a) ((((x) + (a) - 1) / (a)) * (a)) +#define MA_DR_FLAC_CUESHEET_TRACK_PADDED_SIZE_IN_BYTES ma_dr_flac_align(MA_DR_FLAC_CUESHEET_TRACK_SIZE_IN_BYTES, sizeof(ma_uint64)) MA_API void ma_dr_flac_version(ma_uint32* pMajor, ma_uint32* pMinor, ma_uint32* pRevision) { if (pMajor) { @@ -85680,38 +86038,44 @@ static ma_bool32 ma_dr_flac__read_int32(ma_dr_flac_bs* bs, unsigned int bitCount *pResult = (ma_int32)result; return MA_TRUE; } -#ifdef MA_64BIT static ma_bool32 ma_dr_flac__read_uint64(ma_dr_flac_bs* bs, unsigned int bitCount, ma_uint64* pResultOut) { - ma_uint32 resultHi; + ma_uint32 resultHi = 0; ma_uint32 resultLo; + MA_DR_FLAC_ASSERT(bs != NULL); + MA_DR_FLAC_ASSERT(pResultOut != NULL); + MA_DR_FLAC_ASSERT(bitCount > 0); MA_DR_FLAC_ASSERT(bitCount <= 64); - MA_DR_FLAC_ASSERT(bitCount > 32); - if (!ma_dr_flac__read_uint32(bs, bitCount - 32, &resultHi)) { - return MA_FALSE; + if (bitCount > 32) { + if (!ma_dr_flac__read_uint32(bs, bitCount - 32, &resultHi)) { + return MA_FALSE; + } + bitCount = 32; } - if (!ma_dr_flac__read_uint32(bs, 32, &resultLo)) { + if (!ma_dr_flac__read_uint32(bs, bitCount, &resultLo)) { return MA_FALSE; } *pResultOut = (((ma_uint64)resultHi) << 32) | ((ma_uint64)resultLo); return MA_TRUE; } -#endif -#if 0 -static ma_bool32 ma_dr_flac__read_int64(ma_dr_flac_bs* bs, unsigned int bitCount, ma_int64* pResultOut) +static MA_INLINE ma_bool32 ma_dr_flac__read_int64(ma_dr_flac_bs* bs, unsigned int bitCount, ma_int64* pResultOut) { ma_uint64 result; - ma_uint64 signbit; + MA_DR_FLAC_ASSERT(bs != NULL); + MA_DR_FLAC_ASSERT(pResultOut != NULL); + MA_DR_FLAC_ASSERT(bitCount > 0); MA_DR_FLAC_ASSERT(bitCount <= 64); if (!ma_dr_flac__read_uint64(bs, bitCount, &result)) { return MA_FALSE; } - signbit = ((result >> (bitCount-1)) & 0x01); - result |= (~signbit + 1) << bitCount; + if (bitCount < 64) { + ma_uint64 signbit; + signbit = ((result >> (bitCount-1)) & 0x01); + result |= (~signbit + 1) << bitCount; + } *pResultOut = (ma_int64)result; return MA_TRUE; } -#endif static ma_bool32 ma_dr_flac__read_uint16(ma_dr_flac_bs* bs, unsigned int bitCount, ma_uint16* pResult) { ma_uint32 result; @@ -85825,17 +86189,26 @@ static ma_bool32 ma_dr_flac__find_and_seek_to_next_sync_code(ma_dr_flac_bs* bs) if (!ma_dr_flac__read_uint8(bs, 8, &hi)) { return MA_FALSE; } - if (hi == 0xFF) { + while (hi == 0xFF) { ma_uint8 lo; + ma_uint8 trailingBits; if (!ma_dr_flac__read_uint8(bs, 6, &lo)) { return MA_FALSE; } if (lo == 0x3E) { return MA_TRUE; - } else { - if (!ma_dr_flac__seek_bits(bs, MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs) & 7)) { - return MA_FALSE; + } + if (!ma_dr_flac__read_uint8(bs, 2, &trailingBits)) { + return MA_FALSE; + } + hi = (ma_uint8)((lo << 2) | trailingBits); + if (hi == 0xFF) { + #ifndef MA_DR_FLAC_NO_CRC + { + ma_dr_flac__reset_crc16(bs); + bs->crc16 = ma_dr_flac_crc16_byte(0, hi); } + #endif } } } @@ -86035,7 +86408,6 @@ static MA_INLINE ma_bool32 ma_dr_flac__seek_past_next_set_bit(ma_dr_flac_bs* bs, static ma_bool32 ma_dr_flac__seek_to_byte(ma_dr_flac_bs* bs, ma_uint64 offsetFromStart) { MA_DR_FLAC_ASSERT(bs != NULL); - MA_DR_FLAC_ASSERT(offsetFromStart > 0); if (offsetFromStart > 0x7FFFFFFF) { ma_uint64 bytesRemaining = offsetFromStart; if (!bs->onSeek(bs->pUserData, 0x7FFFFFFF, MA_DR_FLAC_SEEK_SET)) { @@ -86130,7 +86502,7 @@ static MA_INLINE ma_bool32 ma_dr_flac__use_64_bit_prediction(ma_uint32 bitsPerSa #if defined(__clang__) __attribute__((no_sanitize("signed-integer-overflow"))) #endif -static MA_INLINE ma_int32 ma_dr_flac__calculate_prediction_32(ma_uint32 order, ma_int32 shift, const ma_int32* coefficients, ma_int32* pDecodedSamples) +static MA_INLINE ma_int32 ma_dr_flac__calculate_prediction_32_s32(ma_uint32 order, ma_int32 shift, const ma_int32* coefficients, ma_int32* pDecodedSamples) { ma_int32 prediction = 0; MA_DR_FLAC_ASSERT(order <= 32); @@ -86171,7 +86543,7 @@ static MA_INLINE ma_int32 ma_dr_flac__calculate_prediction_32(ma_uint32 order, m } return (ma_int32)(prediction >> shift); } -static MA_INLINE ma_int32 ma_dr_flac__calculate_prediction_64(ma_uint32 order, ma_int32 shift, const ma_int32* coefficients, ma_int32* pDecodedSamples) +static MA_INLINE ma_int32 ma_dr_flac__calculate_prediction_64_s32(ma_uint32 order, ma_int32 shift, const ma_int32* coefficients, ma_int32* pDecodedSamples) { ma_int64 prediction; MA_DR_FLAC_ASSERT(order <= 32); @@ -86373,9 +86745,9 @@ static ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__reference(ma_dr decodedRice = (decodedRice >> 1); } if (ma_dr_flac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) { - pSamplesOut[i] = decodedRice + ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + i); + pSamplesOut[i] = decodedRice + ma_dr_flac__calculate_prediction_64_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + i); } else { - pSamplesOut[i] = decodedRice + ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + i); + pSamplesOut[i] = decodedRice + ma_dr_flac__calculate_prediction_32_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + i); } } return MA_TRUE; @@ -86681,10 +87053,10 @@ static ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__scalar(ma_dr_fl riceParamPart1 = (riceParamPart1 >> 1) ^ t[riceParamPart1 & 0x01]; riceParamPart2 = (riceParamPart2 >> 1) ^ t[riceParamPart2 & 0x01]; riceParamPart3 = (riceParamPart3 >> 1) ^ t[riceParamPart3 & 0x01]; - pSamplesOut[0] = riceParamPart0 + ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 0); - pSamplesOut[1] = riceParamPart1 + ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 1); - pSamplesOut[2] = riceParamPart2 + ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 2); - pSamplesOut[3] = riceParamPart3 + ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 3); + pSamplesOut[0] = riceParamPart0 + ma_dr_flac__calculate_prediction_64_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + 0); + pSamplesOut[1] = riceParamPart1 + ma_dr_flac__calculate_prediction_64_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + 1); + pSamplesOut[2] = riceParamPart2 + ma_dr_flac__calculate_prediction_64_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + 2); + pSamplesOut[3] = riceParamPart3 + ma_dr_flac__calculate_prediction_64_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + 3); pSamplesOut += 4; } } else { @@ -86707,10 +87079,10 @@ static ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__scalar(ma_dr_fl riceParamPart1 = (riceParamPart1 >> 1) ^ t[riceParamPart1 & 0x01]; riceParamPart2 = (riceParamPart2 >> 1) ^ t[riceParamPart2 & 0x01]; riceParamPart3 = (riceParamPart3 >> 1) ^ t[riceParamPart3 & 0x01]; - pSamplesOut[0] = riceParamPart0 + ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 0); - pSamplesOut[1] = riceParamPart1 + ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 1); - pSamplesOut[2] = riceParamPart2 + ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 2); - pSamplesOut[3] = riceParamPart3 + ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 3); + pSamplesOut[0] = riceParamPart0 + ma_dr_flac__calculate_prediction_32_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + 0); + pSamplesOut[1] = riceParamPart1 + ma_dr_flac__calculate_prediction_32_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + 1); + pSamplesOut[2] = riceParamPart2 + ma_dr_flac__calculate_prediction_32_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + 2); + pSamplesOut[3] = riceParamPart3 + ma_dr_flac__calculate_prediction_32_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + 3); pSamplesOut += 4; } } @@ -86723,9 +87095,9 @@ static ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__scalar(ma_dr_fl riceParamPart0 |= (zeroCountPart0 << riceParam); riceParamPart0 = (riceParamPart0 >> 1) ^ t[riceParamPart0 & 0x01]; if (ma_dr_flac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) { - pSamplesOut[0] = riceParamPart0 + ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 0); + pSamplesOut[0] = riceParamPart0 + ma_dr_flac__calculate_prediction_64_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + 0); } else { - pSamplesOut[0] = riceParamPart0 + ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 0); + pSamplesOut[0] = riceParamPart0 + ma_dr_flac__calculate_prediction_32_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + 0); } i += 1; pSamplesOut += 1; @@ -86915,7 +87287,7 @@ static ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__sse41_32(ma_dr_ riceParamParts0 &= riceParamMask; riceParamParts0 |= (zeroCountParts0 << riceParam); riceParamParts0 = (riceParamParts0 >> 1) ^ t[riceParamParts0 & 0x01]; - pDecodedSamples[0] = riceParamParts0 + ma_dr_flac__calculate_prediction_32(order, shift, coefficients, pDecodedSamples); + pDecodedSamples[0] = riceParamParts0 + ma_dr_flac__calculate_prediction_32_s32(order, shift, coefficients, pDecodedSamples); i += 1; pDecodedSamples += 1; } @@ -87064,7 +87436,7 @@ static ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__sse41_64(ma_dr_ riceParamParts0 &= riceParamMask; riceParamParts0 |= (zeroCountParts0 << riceParam); riceParamParts0 = (riceParamParts0 >> 1) ^ t[riceParamParts0 & 0x01]; - pDecodedSamples[0] = riceParamParts0 + ma_dr_flac__calculate_prediction_64(order, shift, coefficients, pDecodedSamples); + pDecodedSamples[0] = riceParamParts0 + ma_dr_flac__calculate_prediction_64_s32(order, shift, coefficients, pDecodedSamples); i += 1; pDecodedSamples += 1; } @@ -87282,7 +87654,7 @@ static ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__neon_32(ma_dr_f riceParamParts[0] &= riceParamMask; riceParamParts[0] |= (zeroCountParts[0] << riceParam); riceParamParts[0] = (riceParamParts[0] >> 1) ^ t[riceParamParts[0] & 0x01]; - pDecodedSamples[0] = riceParamParts[0] + ma_dr_flac__calculate_prediction_32(order, shift, coefficients, pDecodedSamples); + pDecodedSamples[0] = riceParamParts[0] + ma_dr_flac__calculate_prediction_32_s32(order, shift, coefficients, pDecodedSamples); i += 1; pDecodedSamples += 1; } @@ -87306,7 +87678,7 @@ static ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__neon_64(ma_dr_f int32x4_t riceParam128; int64x1_t shift64; uint32x4_t one128; - int64x2_t prediction128 = { 0 }; + int64x2_t prediction128; uint32x4_t zeroCountPart128; uint32x4_t riceParamPart128; const ma_uint32 t[2] = {0x00000000, 0xFFFFFFFF}; @@ -87379,7 +87751,7 @@ static ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__neon_64(ma_dr_f riceParamPart128 = veorq_u32(vshrq_n_u32(riceParamPart128, 1), vaddq_u32(ma_dr_flac__vnotq_u32(vandq_u32(riceParamPart128, one128)), one128)); for (i = 0; i < 4; i += 1) { int64x1_t prediction64; - prediction128 = veorq_s64(prediction128, prediction128); + prediction128 = vdupq_n_s64(0); switch (order) { case 12: @@ -87414,7 +87786,7 @@ static ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__neon_64(ma_dr_f riceParamParts[0] &= riceParamMask; riceParamParts[0] |= (zeroCountParts[0] << riceParam); riceParamParts[0] = (riceParamParts[0] >> 1) ^ t[riceParamParts[0] & 0x01]; - pDecodedSamples[0] = riceParamParts[0] + ma_dr_flac__calculate_prediction_64(order, shift, coefficients, pDecodedSamples); + pDecodedSamples[0] = riceParamParts[0] + ma_dr_flac__calculate_prediction_64_s32(order, shift, coefficients, pDecodedSamples); i += 1; pDecodedSamples += 1; } @@ -87483,14 +87855,85 @@ static ma_bool32 ma_dr_flac__decode_samples_with_residual__unencoded(ma_dr_flac_ pSamplesOut[i] = 0; } if (ma_dr_flac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) { - pSamplesOut[i] += ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + i); + pSamplesOut[i] += ma_dr_flac__calculate_prediction_64_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + i); } else { - pSamplesOut[i] += ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + i); + pSamplesOut[i] += ma_dr_flac__calculate_prediction_32_s32(lpcOrder, lpcShift, coefficients, pSamplesOut + i); } } return MA_TRUE; } -static ma_bool32 ma_dr_flac__decode_samples_with_residual(ma_dr_flac_bs* bs, ma_uint32 bitsPerSample, ma_uint32 blockSize, ma_uint32 lpcOrder, ma_int32 lpcShift, ma_uint32 lpcPrecision, const ma_int32* coefficients, ma_int32* pDecodedSamples) +static MA_INLINE ma_int64 ma_dr_flac__calculate_prediction_64_s64(ma_uint32 order, ma_int32 shift, const ma_int32* coefficients, const ma_int64* pDecodedSamples) +{ + ma_int64 prediction = 0; + ma_uint32 i; + for (i = 0; i < order; ++i) { + prediction += coefficients[i] * pDecodedSamples[-(ma_int32)i - 1]; + } + return prediction >> shift; +} +static ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__s64(ma_dr_flac_bs* bs, ma_uint32 bitsPerSample, ma_uint32 count, ma_uint8 riceParam, ma_uint32 lpcOrder, ma_int32 lpcShift, const ma_int32* coefficients, ma_int64* pSamplesOut) +{ + ma_uint32 riceParamMask; + ma_int64 minSample; + ma_int64 maxSample; + ma_uint32 i; + MA_DR_FLAC_ASSERT(bitsPerSample > 0 && bitsPerSample <= 33); + riceParamMask = (1U << riceParam) - 1; + minSample = -((ma_int64)1 << (bitsPerSample - 1)); + maxSample = -(minSample + 1); + for (i = 0; i < count; ++i) { + ma_uint32 zeroCountPart; + ma_uint32 riceParamPart; + ma_int32 residual; + ma_int64 sample; + if (!ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountPart, &riceParamPart)) { + return MA_FALSE; + } + if (zeroCountPart > (0xFFFFFFFFU >> riceParam)) { + return MA_FALSE; + } + riceParamPart = (zeroCountPart << riceParam) | (riceParamPart & riceParamMask); + if (riceParamPart == 0xFFFFFFFFU) { + return MA_FALSE; + } + residual = (ma_int32)(riceParamPart >> 1); + if ((riceParamPart & 1) != 0) { + residual = -residual - 1; + } + sample = ma_dr_flac__calculate_prediction_64_s64(lpcOrder, lpcShift, coefficients, pSamplesOut + i) + residual; + if (sample < minSample || sample > maxSample) { + return MA_FALSE; + } + pSamplesOut[i] = sample; + } + return MA_TRUE; +} +static ma_bool32 ma_dr_flac__decode_samples_with_residual__unencoded_s64(ma_dr_flac_bs* bs, ma_uint32 bitsPerSample, ma_uint32 count, ma_uint8 unencodedBitsPerSample, ma_uint32 lpcOrder, ma_int32 lpcShift, const ma_int32* coefficients, ma_int64* pSamplesOut) +{ + ma_int64 minSample; + ma_int64 maxSample; + ma_uint32 i; + MA_DR_FLAC_ASSERT(bitsPerSample > 0 && bitsPerSample <= 33); + MA_DR_FLAC_ASSERT(unencodedBitsPerSample <= 31); + minSample = -((ma_int64)1 << (bitsPerSample - 1)); + maxSample = -(minSample + 1); + for (i = 0; i < count; ++i) { + ma_int32 residual = 0; + ma_int64 sample; + if (unencodedBitsPerSample > 0) { + if (!ma_dr_flac__read_int32(bs, unencodedBitsPerSample, &residual)) { + return MA_FALSE; + } + } + sample = ma_dr_flac__calculate_prediction_64_s64(lpcOrder, lpcShift, coefficients, pSamplesOut + i) + residual; + if (sample < minSample || sample > maxSample) { + return MA_FALSE; + } + pSamplesOut[i] = sample; + } + return MA_TRUE; +} +static ma_bool32 ma_dr_flac__decode_samples_with_residual(ma_dr_flac_bs* bs, ma_uint32 bitsPerSample, ma_uint32 blockSize, ma_uint32 lpcOrder, ma_int32 lpcShift, ma_uint32 lpcPrecision, const ma_int32* coefficients, ma_int32* pDecodedSamplesS32, ma_int64* pDecodedSamplesS64) { ma_uint8 residualMethod; ma_uint8 partitionOrder; @@ -87498,23 +87941,28 @@ static ma_bool32 ma_dr_flac__decode_samples_with_residual(ma_dr_flac_bs* bs, ma_ ma_uint32 partitionsRemaining; MA_DR_FLAC_ASSERT(bs != NULL); MA_DR_FLAC_ASSERT(blockSize != 0); - MA_DR_FLAC_ASSERT(pDecodedSamples != NULL); + MA_DR_FLAC_ASSERT(pDecodedSamplesS32 != NULL || pDecodedSamplesS64 != NULL); + MA_DR_FLAC_ASSERT(pDecodedSamplesS32 == NULL || pDecodedSamplesS64 == NULL); if (!ma_dr_flac__read_uint8(bs, 2, &residualMethod)) { return MA_FALSE; } if (residualMethod != MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE && residualMethod != MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2) { return MA_FALSE; } - pDecodedSamples += lpcOrder; if (!ma_dr_flac__read_uint8(bs, 4, &partitionOrder)) { return MA_FALSE; } - if (partitionOrder > 8) { + if ((blockSize & ((1U << partitionOrder) - 1)) != 0) { return MA_FALSE; } if ((blockSize / (1 << partitionOrder)) < lpcOrder) { return MA_FALSE; } + if (pDecodedSamplesS32 != NULL) { + pDecodedSamplesS32 += lpcOrder; + } else { + pDecodedSamplesS64 += lpcOrder; + } samplesInPartition = (blockSize / (1 << partitionOrder)) - lpcOrder; partitionsRemaining = (1 << partitionOrder); for (;;) { @@ -87535,19 +87983,35 @@ static ma_bool32 ma_dr_flac__decode_samples_with_residual(ma_dr_flac_bs* bs, ma_ } } if (riceParam != 0xFF) { - if (!ma_dr_flac__decode_samples_with_residual__rice(bs, bitsPerSample, samplesInPartition, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pDecodedSamples)) { - return MA_FALSE; + if (pDecodedSamplesS32 != NULL) { + if (!ma_dr_flac__decode_samples_with_residual__rice(bs, bitsPerSample, samplesInPartition, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pDecodedSamplesS32)) { + return MA_FALSE; + } + } else { + if (!ma_dr_flac__decode_samples_with_residual__rice__s64(bs, bitsPerSample, samplesInPartition, riceParam, lpcOrder, lpcShift, coefficients, pDecodedSamplesS64)) { + return MA_FALSE; + } } } else { ma_uint8 unencodedBitsPerSample = 0; if (!ma_dr_flac__read_uint8(bs, 5, &unencodedBitsPerSample)) { return MA_FALSE; } - if (!ma_dr_flac__decode_samples_with_residual__unencoded(bs, bitsPerSample, samplesInPartition, unencodedBitsPerSample, lpcOrder, lpcShift, lpcPrecision, coefficients, pDecodedSamples)) { - return MA_FALSE; + if (pDecodedSamplesS32 != NULL) { + if (!ma_dr_flac__decode_samples_with_residual__unencoded(bs, bitsPerSample, samplesInPartition, unencodedBitsPerSample, lpcOrder, lpcShift, lpcPrecision, coefficients, pDecodedSamplesS32)) { + return MA_FALSE; + } + } else { + if (!ma_dr_flac__decode_samples_with_residual__unencoded_s64(bs, bitsPerSample, samplesInPartition, unencodedBitsPerSample, lpcOrder, lpcShift, coefficients, pDecodedSamplesS64)) { + return MA_FALSE; + } } } - pDecodedSamples += samplesInPartition; + if (pDecodedSamplesS32 != NULL) { + pDecodedSamplesS32 += samplesInPartition; + } else { + pDecodedSamplesS64 += samplesInPartition; + } if (partitionsRemaining == 1) { break; } @@ -87575,10 +88039,10 @@ static ma_bool32 ma_dr_flac__read_and_seek_residual(ma_dr_flac_bs* bs, ma_uint32 if (!ma_dr_flac__read_uint8(bs, 4, &partitionOrder)) { return MA_FALSE; } - if (partitionOrder > 8) { + if ((blockSize & ((1U << partitionOrder) - 1)) != 0) { return MA_FALSE; } - if ((blockSize / (1 << partitionOrder)) <= order) { + if ((blockSize / (1 << partitionOrder)) < order) { return MA_FALSE; } samplesInPartition = (blockSize / (1 << partitionOrder)) - order; @@ -87622,31 +88086,51 @@ static ma_bool32 ma_dr_flac__read_and_seek_residual(ma_dr_flac_bs* bs, ma_uint32 } return MA_TRUE; } -static ma_bool32 ma_dr_flac__decode_samples__constant(ma_dr_flac_bs* bs, ma_uint32 blockSize, ma_uint32 subframeBitsPerSample, ma_int32* pDecodedSamples) +static ma_bool32 ma_dr_flac__decode_samples__constant(ma_dr_flac_bs* bs, ma_uint32 blockSize, ma_uint32 subframeBitsPerSample, ma_int32* pDecodedSamplesS32, ma_int64* pDecodedSamplesS64) { ma_uint32 i; - ma_int32 sample; - if (!ma_dr_flac__read_int32(bs, subframeBitsPerSample, &sample)) { - return MA_FALSE; - } - for (i = 0; i < blockSize; ++i) { - pDecodedSamples[i] = sample; + MA_DR_FLAC_ASSERT(pDecodedSamplesS32 != NULL || pDecodedSamplesS64 != NULL); + MA_DR_FLAC_ASSERT(pDecodedSamplesS32 == NULL || pDecodedSamplesS64 == NULL); + if (pDecodedSamplesS32 != NULL) { + ma_int32 sample; + if (!ma_dr_flac__read_int32(bs, subframeBitsPerSample, &sample)) { + return MA_FALSE; + } + for (i = 0; i < blockSize; ++i) { + pDecodedSamplesS32[i] = sample; + } + } else { + ma_int64 sample; + if (!ma_dr_flac__read_int64(bs, subframeBitsPerSample, &sample)) { + return MA_FALSE; + } + for (i = 0; i < blockSize; ++i) { + pDecodedSamplesS64[i] = sample; + } } return MA_TRUE; } -static ma_bool32 ma_dr_flac__decode_samples__verbatim(ma_dr_flac_bs* bs, ma_uint32 blockSize, ma_uint32 subframeBitsPerSample, ma_int32* pDecodedSamples) +static ma_bool32 ma_dr_flac__decode_samples__verbatim(ma_dr_flac_bs* bs, ma_uint32 blockSize, ma_uint32 subframeBitsPerSample, ma_int32* pDecodedSamplesS32, ma_int64* pDecodedSamplesS64) { ma_uint32 i; - for (i = 0; i < blockSize; ++i) { - ma_int32 sample; - if (!ma_dr_flac__read_int32(bs, subframeBitsPerSample, &sample)) { - return MA_FALSE; + MA_DR_FLAC_ASSERT(pDecodedSamplesS32 != NULL || pDecodedSamplesS64 != NULL); + MA_DR_FLAC_ASSERT(pDecodedSamplesS32 == NULL || pDecodedSamplesS64 == NULL); + if (pDecodedSamplesS32 != NULL) { + for (i = 0; i < blockSize; ++i) { + if (!ma_dr_flac__read_int32(bs, subframeBitsPerSample, pDecodedSamplesS32 + i)) { + return MA_FALSE; + } + } + } else { + for (i = 0; i < blockSize; ++i) { + if (!ma_dr_flac__read_int64(bs, subframeBitsPerSample, pDecodedSamplesS64 + i)) { + return MA_FALSE; + } } - pDecodedSamples[i] = sample; } return MA_TRUE; } -static ma_bool32 ma_dr_flac__decode_samples__fixed(ma_dr_flac_bs* bs, ma_uint32 blockSize, ma_uint32 subframeBitsPerSample, ma_uint8 lpcOrder, ma_int32* pDecodedSamples) +static ma_bool32 ma_dr_flac__decode_samples__fixed(ma_dr_flac_bs* bs, ma_uint32 blockSize, ma_uint32 subframeBitsPerSample, ma_uint8 lpcOrder, ma_int32* pDecodedSamplesS32, ma_int64* pDecodedSamplesS64) { ma_uint32 i; static ma_int32 lpcCoefficientsTable[5][4] = { @@ -87656,30 +88140,46 @@ static ma_bool32 ma_dr_flac__decode_samples__fixed(ma_dr_flac_bs* bs, ma_uint32 {3, -3, 1, 0}, {4, -6, 4, -1} }; - for (i = 0; i < lpcOrder; ++i) { - ma_int32 sample; - if (!ma_dr_flac__read_int32(bs, subframeBitsPerSample, &sample)) { - return MA_FALSE; + MA_DR_FLAC_ASSERT(pDecodedSamplesS32 != NULL || pDecodedSamplesS64 != NULL); + MA_DR_FLAC_ASSERT(pDecodedSamplesS32 == NULL || pDecodedSamplesS64 == NULL); + if (pDecodedSamplesS32 != NULL) { + for (i = 0; i < lpcOrder; ++i) { + if (!ma_dr_flac__read_int32(bs, subframeBitsPerSample, pDecodedSamplesS32 + i)) { + return MA_FALSE; + } + } + } else { + for (i = 0; i < lpcOrder; ++i) { + if (!ma_dr_flac__read_int64(bs, subframeBitsPerSample, pDecodedSamplesS64 + i)) { + return MA_FALSE; + } } - pDecodedSamples[i] = sample; } - if (!ma_dr_flac__decode_samples_with_residual(bs, subframeBitsPerSample, blockSize, lpcOrder, 0, 4, lpcCoefficientsTable[lpcOrder], pDecodedSamples)) { + if (!ma_dr_flac__decode_samples_with_residual(bs, subframeBitsPerSample, blockSize, lpcOrder, 0, 4, lpcCoefficientsTable[lpcOrder], pDecodedSamplesS32, pDecodedSamplesS64)) { return MA_FALSE; } return MA_TRUE; } -static ma_bool32 ma_dr_flac__decode_samples__lpc(ma_dr_flac_bs* bs, ma_uint32 blockSize, ma_uint32 bitsPerSample, ma_uint8 lpcOrder, ma_int32* pDecodedSamples) +static ma_bool32 ma_dr_flac__decode_samples__lpc(ma_dr_flac_bs* bs, ma_uint32 blockSize, ma_uint32 bitsPerSample, ma_uint8 lpcOrder, ma_int32* pDecodedSamplesS32, ma_int64* pDecodedSamplesS64) { ma_uint8 i; ma_uint8 lpcPrecision; ma_int8 lpcShift; ma_int32 coefficients[32]; - for (i = 0; i < lpcOrder; ++i) { - ma_int32 sample; - if (!ma_dr_flac__read_int32(bs, bitsPerSample, &sample)) { - return MA_FALSE; + MA_DR_FLAC_ASSERT(pDecodedSamplesS32 != NULL || pDecodedSamplesS64 != NULL); + MA_DR_FLAC_ASSERT(pDecodedSamplesS32 == NULL || pDecodedSamplesS64 == NULL); + if (pDecodedSamplesS32 != NULL) { + for (i = 0; i < lpcOrder; ++i) { + if (!ma_dr_flac__read_int32(bs, bitsPerSample, pDecodedSamplesS32 + i)) { + return MA_FALSE; + } + } + } else { + for (i = 0; i < lpcOrder; ++i) { + if (!ma_dr_flac__read_int64(bs, bitsPerSample, pDecodedSamplesS64 + i)) { + return MA_FALSE; + } } - pDecodedSamples[i] = sample; } if (!ma_dr_flac__read_uint8(bs, 4, &lpcPrecision)) { return MA_FALSE; @@ -87700,7 +88200,7 @@ static ma_bool32 ma_dr_flac__decode_samples__lpc(ma_dr_flac_bs* bs, ma_uint32 bl return MA_FALSE; } } - if (!ma_dr_flac__decode_samples_with_residual(bs, bitsPerSample, blockSize, lpcOrder, lpcShift, lpcPrecision, coefficients, pDecodedSamples)) { + if (!ma_dr_flac__decode_samples_with_residual(bs, bitsPerSample, blockSize, lpcOrder, lpcShift, lpcPrecision, coefficients, pDecodedSamplesS32, pDecodedSamplesS64)) { return MA_FALSE; } return MA_TRUE; @@ -87708,7 +88208,7 @@ static ma_bool32 ma_dr_flac__decode_samples__lpc(ma_dr_flac_bs* bs, ma_uint32 bl static ma_bool32 ma_dr_flac__read_next_flac_frame_header(ma_dr_flac_bs* bs, ma_uint8 streaminfoBitsPerSample, ma_dr_flac_frame_header* header) { const ma_uint32 sampleRateTable[12] = {0, 88200, 176400, 192000, 8000, 16000, 22050, 24000, 32000, 44100, 48000, 96000}; - const ma_uint8 bitsPerSampleTable[8] = {0, 8, 12, (ma_uint8)-1, 16, 20, 24, (ma_uint8)-1}; + const ma_uint8 bitsPerSampleTable[8] = {0, 8, 12, (ma_uint8)-1, 16, 20, 24, 32}; MA_DR_FLAC_ASSERT(bs != NULL); MA_DR_FLAC_ASSERT(header != NULL); for (;;) { @@ -87755,7 +88255,7 @@ static ma_bool32 ma_dr_flac__read_next_flac_frame_header(ma_dr_flac_bs* bs, ma_u if (!ma_dr_flac__read_uint8(bs, 3, &bitsPerSample)) { return MA_FALSE; } - if (bitsPerSample == 3 || bitsPerSample == 7) { + if (bitsPerSample == 3) { continue; } crc8 = ma_dr_flac_crc8(crc8, bitsPerSample, 3); @@ -87844,8 +88344,16 @@ static ma_bool32 ma_dr_flac__read_next_flac_frame_header(ma_dr_flac_bs* bs, ma_u if (header->bitsPerSample == 0) { header->bitsPerSample = streaminfoBitsPerSample; } - if (header->bitsPerSample != streaminfoBitsPerSample) { - return MA_FALSE; + if (streaminfoBitsPerSample != 0) { + if (streaminfoBitsPerSample < 32) { + if (header->bitsPerSample == 32) { + return MA_FALSE; + } + } else { + if (header->bitsPerSample != 32) { + return MA_FALSE; + } + } } if (!ma_dr_flac__read_uint8(bs, 8, &header->crc8)) { return MA_FALSE; @@ -87902,12 +88410,15 @@ static ma_bool32 ma_dr_flac__read_subframe_header(ma_dr_flac_bs* bs, ma_dr_flac_ } return MA_TRUE; } -static ma_bool32 ma_dr_flac__decode_subframe(ma_dr_flac_bs* bs, ma_dr_flac_frame* frame, int subframeIndex, ma_int32* pDecodedSamplesOut) +static ma_bool32 ma_dr_flac__decode_subframe(ma_dr_flac_bs* bs, ma_dr_flac_frame* frame, int subframeIndex, ma_int32* pDecodedSamplesOutS32, ma_int64* pDecodedSamplesOutS64) { ma_dr_flac_subframe* pSubframe; ma_uint32 subframeBitsPerSample; + ma_bool32 decodeResult; MA_DR_FLAC_ASSERT(bs != NULL); MA_DR_FLAC_ASSERT(frame != NULL); + MA_DR_FLAC_ASSERT(pDecodedSamplesOutS32 != NULL || pDecodedSamplesOutS64 != NULL); + MA_DR_FLAC_ASSERT(pDecodedSamplesOutS32 == NULL || pDecodedSamplesOutS64 == NULL); pSubframe = frame->subframes + subframeIndex; if (!ma_dr_flac__read_subframe_header(bs, pSubframe)) { return MA_FALSE; @@ -87918,14 +88429,19 @@ static ma_bool32 ma_dr_flac__decode_subframe(ma_dr_flac_bs* bs, ma_dr_flac_frame } else if (frame->header.channelAssignment == MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE && subframeIndex == 0) { subframeBitsPerSample += 1; } - if (subframeBitsPerSample > 32) { - return MA_FALSE; - } if (pSubframe->wastedBitsPerSample >= subframeBitsPerSample) { return MA_FALSE; } subframeBitsPerSample -= pSubframe->wastedBitsPerSample; - pSubframe->pSamplesS32 = pDecodedSamplesOut; + if (pDecodedSamplesOutS32 != NULL) { + if (subframeBitsPerSample > 32) { + return MA_FALSE; + } + } else { + if (subframeBitsPerSample > 33) { + return MA_FALSE; + } + } if (frame->header.blockSizeInPCMFrames < pSubframe->lpcOrder) { return MA_FALSE; } @@ -87933,21 +88449,29 @@ static ma_bool32 ma_dr_flac__decode_subframe(ma_dr_flac_bs* bs, ma_dr_flac_frame { case MA_DR_FLAC_SUBFRAME_CONSTANT: { - ma_dr_flac__decode_samples__constant(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->pSamplesS32); + decodeResult = ma_dr_flac__decode_samples__constant(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pDecodedSamplesOutS32, pDecodedSamplesOutS64); } break; case MA_DR_FLAC_SUBFRAME_VERBATIM: { - ma_dr_flac__decode_samples__verbatim(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->pSamplesS32); + decodeResult = ma_dr_flac__decode_samples__verbatim(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pDecodedSamplesOutS32, pDecodedSamplesOutS64); } break; case MA_DR_FLAC_SUBFRAME_FIXED: { - ma_dr_flac__decode_samples__fixed(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->lpcOrder, pSubframe->pSamplesS32); + decodeResult = ma_dr_flac__decode_samples__fixed(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->lpcOrder, pDecodedSamplesOutS32, pDecodedSamplesOutS64); } break; case MA_DR_FLAC_SUBFRAME_LPC: { - ma_dr_flac__decode_samples__lpc(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->lpcOrder, pSubframe->pSamplesS32); + decodeResult = ma_dr_flac__decode_samples__lpc(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->lpcOrder, pDecodedSamplesOutS32, pDecodedSamplesOutS64); } break; - default: return MA_FALSE; + default: decodeResult = MA_FALSE; + } + if (!decodeResult) { + return MA_FALSE; + } + if (pDecodedSamplesOutS32 != NULL) { + pSubframe->samples.pS32 = pDecodedSamplesOutS32; + } else { + pSubframe->samples.pS64 = pDecodedSamplesOutS64; } return MA_TRUE; } @@ -87971,7 +88495,7 @@ static ma_bool32 ma_dr_flac__seek_subframe(ma_dr_flac_bs* bs, ma_dr_flac_frame* return MA_FALSE; } subframeBitsPerSample -= pSubframe->wastedBitsPerSample; - pSubframe->pSamplesS32 = NULL; + MA_DR_FLAC_ZERO_MEMORY(&pSubframe->samples, sizeof(pSubframe->samples)); switch (pSubframe->subframeType) { case MA_DR_FLAC_SUBFRAME_CONSTANT: @@ -88029,6 +88553,7 @@ static MA_INLINE ma_uint8 ma_dr_flac__get_channel_count_from_channel_assignment( MA_DR_FLAC_ASSERT(channelAssignment <= 10); return lookup[channelAssignment]; } +static ma_bool32 ma_dr_flac__resize_decoded_samples(ma_dr_flac* pFlac, ma_uint16 blockSizeInPCMFrames); static ma_result ma_dr_flac__decode_flac_frame(ma_dr_flac* pFlac) { int channelCount; @@ -88038,17 +88563,28 @@ static ma_result ma_dr_flac__decode_flac_frame(ma_dr_flac* pFlac) #ifndef MA_DR_FLAC_NO_CRC ma_uint16 actualCRC16; #endif + pFlac->currentFLACFrame.pcmFramesRemaining = 0; MA_DR_FLAC_ZERO_MEMORY(pFlac->currentFLACFrame.subframes, sizeof(pFlac->currentFLACFrame.subframes)); - if (pFlac->currentFLACFrame.header.blockSizeInPCMFrames > pFlac->maxBlockSizeInPCMFrames) { - return MA_ERROR; - } channelCount = ma_dr_flac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment); if (channelCount != (int)pFlac->channels) { return MA_ERROR; } - for (i = 0; i < channelCount; ++i) { - if (!ma_dr_flac__decode_subframe(&pFlac->bs, &pFlac->currentFLACFrame, i, pFlac->pDecodedSamples + (pFlac->currentFLACFrame.header.blockSizeInPCMFrames * i))) { - return MA_ERROR; + if (pFlac->currentFLACFrame.header.blockSizeInPCMFrames > pFlac->decodedSamplesCapacityInPCMFrames) { + if (!ma_dr_flac__resize_decoded_samples(pFlac, pFlac->currentFLACFrame.header.blockSizeInPCMFrames)) { + return MA_OUT_OF_MEMORY; + } + } + if (pFlac->bitsPerSample != 32) { + for (i = 0; i < channelCount; ++i) { + if (!ma_dr_flac__decode_subframe(&pFlac->bs, &pFlac->currentFLACFrame, i, pFlac->decodedSamples.pS32 + (pFlac->currentFLACFrame.header.blockSizeInPCMFrames * i), NULL)) { + return MA_ERROR; + } + } + } else { + for (i = 0; i < channelCount; ++i) { + if (!ma_dr_flac__decode_subframe(&pFlac->bs, &pFlac->currentFLACFrame, i, NULL, pFlac->decodedSamples.pS64 + (pFlac->currentFLACFrame.header.blockSizeInPCMFrames * i))) { + return MA_ERROR; + } } } paddingSizeInBits = (ma_uint8)(MA_DR_FLAC_CACHE_L1_BITS_REMAINING(&pFlac->bs) & 7); @@ -88074,33 +88610,42 @@ static ma_result ma_dr_flac__decode_flac_frame(ma_dr_flac* pFlac) } static ma_result ma_dr_flac__seek_flac_frame(ma_dr_flac* pFlac) { + ma_result result; int channelCount; int i; ma_uint16 desiredCRC16; #ifndef MA_DR_FLAC_NO_CRC ma_uint16 actualCRC16; #endif + pFlac->currentFLACFrame.pcmFramesRemaining = 0; channelCount = ma_dr_flac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment); for (i = 0; i < channelCount; ++i) { if (!ma_dr_flac__seek_subframe(&pFlac->bs, &pFlac->currentFLACFrame, i)) { - return MA_ERROR; + result = MA_ERROR; + goto error; } } if (!ma_dr_flac__seek_bits(&pFlac->bs, MA_DR_FLAC_CACHE_L1_BITS_REMAINING(&pFlac->bs) & 7)) { - return MA_ERROR; + result = MA_ERROR; + goto error; } #ifndef MA_DR_FLAC_NO_CRC actualCRC16 = ma_dr_flac__flush_crc16(&pFlac->bs); #endif if (!ma_dr_flac__read_uint16(&pFlac->bs, 16, &desiredCRC16)) { - return MA_AT_END; + result = MA_AT_END; + goto error; } #ifndef MA_DR_FLAC_NO_CRC if (actualCRC16 != desiredCRC16) { - return MA_CRC_MISMATCH; + result = MA_CRC_MISMATCH; + goto error; } #endif return MA_SUCCESS; +error: + MA_DR_FLAC_ZERO_MEMORY(pFlac->currentFLACFrame.subframes, sizeof(pFlac->currentFLACFrame.subframes)); + return result; } static ma_bool32 ma_dr_flac__read_and_decode_next_flac_frame(ma_dr_flac* pFlac) { @@ -88108,10 +88653,12 @@ static ma_bool32 ma_dr_flac__read_and_decode_next_flac_frame(ma_dr_flac* pFlac) for (;;) { ma_result result; if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + MA_DR_FLAC_ZERO_MEMORY(pFlac->currentFLACFrame.subframes, sizeof(pFlac->currentFLACFrame.subframes)); return MA_FALSE; } result = ma_dr_flac__decode_flac_frame(pFlac); if (result != MA_SUCCESS) { + MA_DR_FLAC_ZERO_MEMORY(pFlac->currentFLACFrame.subframes, sizeof(pFlac->currentFLACFrame.subframes)); if (result == MA_CRC_MISMATCH) { continue; } else { @@ -88240,7 +88787,7 @@ static ma_bool32 ma_dr_flac__seek_to_pcm_frame__brute_force(ma_dr_flac* pFlac, m pFlac->currentFLACFrame.pcmFramesRemaining = 0; isMidFrame = MA_FALSE; } - if (pcmFrameIndex == pFlac->totalPCMFrameCount && runningPCMFrameCount == pFlac->totalPCMFrameCount) { + if (runningPCMFrameCount == pcmFrameIndex) { return MA_TRUE; } } @@ -88252,6 +88799,23 @@ static ma_bool32 ma_dr_flac__seek_to_pcm_frame__brute_force(ma_dr_flac* pFlac, m } #if !defined(MA_DR_FLAC_NO_CRC) #define MA_DR_FLAC_BINARY_SEARCH_APPROX_COMPRESSION_RATIO 0.6f +static ma_bool32 ma_dr_flac__read_and_decode_next_flac_frame_for_seek(ma_dr_flac* pFlac) +{ + MA_DR_FLAC_ASSERT(pFlac != NULL); + for (;;) { + ma_result result; + if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + return MA_FALSE; + } + result = ma_dr_flac__decode_flac_frame(pFlac); + if (result == MA_SUCCESS) { + return MA_TRUE; + } + if (result == MA_AT_END) { + return MA_FALSE; + } + } +} static ma_bool32 ma_dr_flac__seek_to_approximate_flac_frame_to_byte(ma_dr_flac* pFlac, ma_uint64 targetByte, ma_uint64 rangeLo, ma_uint64 rangeHi, ma_uint64* pLastSuccessfulSeekOffset) { MA_DR_FLAC_ASSERT(pFlac != NULL); @@ -88260,34 +88824,40 @@ static ma_bool32 ma_dr_flac__seek_to_approximate_flac_frame_to_byte(ma_dr_flac* MA_DR_FLAC_ASSERT(targetByte <= rangeHi); *pLastSuccessfulSeekOffset = pFlac->firstFLACFramePosInBytes; for (;;) { - ma_uint64 lastTargetByte = targetByte; if (!ma_dr_flac__seek_to_byte(&pFlac->bs, targetByte)) { if (targetByte == 0) { ma_dr_flac__seek_to_first_frame(pFlac); return MA_FALSE; } - targetByte = rangeLo + ((rangeHi - rangeLo)/2); rangeHi = targetByte; + if (rangeHi <= rangeLo) { + return MA_FALSE; + } + targetByte = rangeLo + ((rangeHi - rangeLo)/2); } else { MA_DR_FLAC_ZERO_MEMORY(&pFlac->currentFLACFrame, sizeof(pFlac->currentFLACFrame)); -#if 1 - if (!ma_dr_flac__read_and_decode_next_flac_frame(pFlac)) { - targetByte = rangeLo + ((rangeHi - rangeLo)/2); - rangeHi = targetByte; - } else { - break; + #if 1 + { + if (!ma_dr_flac__read_and_decode_next_flac_frame_for_seek(pFlac)) { + rangeHi = targetByte; + if (rangeHi <= rangeLo) { + return MA_FALSE; + } + targetByte = rangeLo + ((rangeHi - rangeLo)/2); + } else { + break; + } } -#else - if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { - targetByte = rangeLo + ((rangeHi - rangeLo)/2); - rangeHi = targetByte; - } else { - break; + #else + { + if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + targetByte = rangeLo + ((rangeHi - rangeLo)/2); + rangeHi = targetByte; + } else { + break; + } } -#endif - } - if(targetByte == lastTargetByte) { - return MA_FALSE; + #endif } } ma_dr_flac__get_pcm_frame_range_of_current_flac_frame(pFlac, &pFlac->currentPCMFrame, NULL); @@ -88314,11 +88884,29 @@ static ma_bool32 ma_dr_flac__seek_to_pcm_frame__binary_search_internal(ma_dr_fla ma_uint64 lastSuccessfulSeekOffset = (ma_uint64)-1; ma_uint64 closestSeekOffsetBeforeTargetPCMFrame = byteRangeLo; ma_uint32 seekForwardThreshold = (pFlac->maxBlockSizeInPCMFrames != 0) ? pFlac->maxBlockSizeInPCMFrames*2 : 4096; - targetByte = byteRangeLo + (ma_uint64)(((ma_int64)((pcmFrameIndex - pFlac->currentPCMFrame) * pFlac->channels * pFlac->bitsPerSample)/8.0f) * MA_DR_FLAC_BINARY_SEARCH_APPROX_COMPRESSION_RATIO); - if (targetByte > byteRangeHi) { + if (byteRangeLo > byteRangeHi || pFlac->currentPCMFrame > pcmFrameIndex) { + return MA_FALSE; + } + targetByte = (ma_uint64)(((ma_int64)((pcmFrameIndex - pFlac->currentPCMFrame) * pFlac->channels * pFlac->bitsPerSample)/8.0f) * MA_DR_FLAC_BINARY_SEARCH_APPROX_COMPRESSION_RATIO); + if (targetByte > byteRangeHi - byteRangeLo) { targetByte = byteRangeHi; + } else { + targetByte += byteRangeLo; } for (;;) { + ma_uint64 previousByteRangeLo; + ma_uint64 previousByteRangeHi; + previousByteRangeLo = byteRangeLo; + previousByteRangeHi = byteRangeHi; + if ((byteRangeHi - byteRangeLo) <= 1) { + if (!ma_dr_flac__seek_to_approximate_flac_frame_to_byte(pFlac, closestSeekOffsetBeforeTargetPCMFrame, closestSeekOffsetBeforeTargetPCMFrame, byteRangeHi, &lastSuccessfulSeekOffset)) { + break; + } + if (pFlac->currentPCMFrame <= pcmFrameIndex && ma_dr_flac__decode_flac_frame_and_seek_forward_by_pcm_frames(pFlac, pcmFrameIndex - pFlac->currentPCMFrame)) { + return MA_TRUE; + } + break; + } if (ma_dr_flac__seek_to_approximate_flac_frame_to_byte(pFlac, targetByte, byteRangeLo, byteRangeHi, &lastSuccessfulSeekOffset)) { ma_uint64 newPCMRangeLo; ma_uint64 newPCMRangeHi; @@ -88342,7 +88930,6 @@ static ma_bool32 ma_dr_flac__seek_to_pcm_frame__binary_search_internal(ma_dr_fla break; } } else { - const float approxCompressionRatio = (ma_int64)(lastSuccessfulSeekOffset - pFlac->firstFLACFramePosInBytes) / ((ma_int64)(pcmRangeLo * pFlac->channels * pFlac->bitsPerSample)/8.0f); if (pcmRangeLo > pcmFrameIndex) { byteRangeHi = lastSuccessfulSeekOffset; if (byteRangeLo > byteRangeHi) { @@ -88360,11 +88947,12 @@ static ma_bool32 ma_dr_flac__seek_to_pcm_frame__binary_search_internal(ma_dr_fla break; } } else { + const double approxCompressionRatio = (ma_int64)(lastSuccessfulSeekOffset - pFlac->firstFLACFramePosInBytes) / ((ma_int64)(pcmRangeLo * pFlac->channels * pFlac->bitsPerSample)/8.0); byteRangeLo = lastSuccessfulSeekOffset; if (byteRangeHi < byteRangeLo) { byteRangeHi = byteRangeLo; } - targetByte = lastSuccessfulSeekOffset + (ma_uint64)(((ma_int64)((pcmFrameIndex-pcmRangeLo) * pFlac->channels * pFlac->bitsPerSample)/8.0f) * approxCompressionRatio); + targetByte = lastSuccessfulSeekOffset + (ma_uint64)(((ma_int64)((pcmFrameIndex-pcmRangeLo) * pFlac->channels * pFlac->bitsPerSample)/8.0) * approxCompressionRatio); if (targetByte > byteRangeHi) { targetByte = byteRangeHi; } @@ -88374,6 +88962,18 @@ static ma_bool32 ma_dr_flac__seek_to_pcm_frame__binary_search_internal(ma_dr_fla } } } + if (byteRangeLo == previousByteRangeLo && byteRangeHi == previousByteRangeHi) { + if (!ma_dr_flac__seek_to_approximate_flac_frame_to_byte(pFlac, closestSeekOffsetBeforeTargetPCMFrame, closestSeekOffsetBeforeTargetPCMFrame, byteRangeHi, &lastSuccessfulSeekOffset)) { + break; + } + if (pFlac->currentPCMFrame <= pcmFrameIndex && ma_dr_flac__decode_flac_frame_and_seek_forward_by_pcm_frames(pFlac, pcmFrameIndex - pFlac->currentPCMFrame)) { + return MA_TRUE; + } + break; + } + if ((byteRangeHi - byteRangeLo) > 1 && (targetByte <= byteRangeLo || targetByte >= byteRangeHi)) { + targetByte = byteRangeLo + ((byteRangeHi - byteRangeLo) / 2); + } } else { break; } @@ -88422,6 +89022,9 @@ static ma_bool32 ma_dr_flac__seek_to_pcm_frame__seek_table(ma_dr_flac* pFlac, ma if (pFlac->pSeekpoints[iClosestSeekpoint].firstPCMFrame > pFlac->totalPCMFrameCount && pFlac->totalPCMFrameCount > 0) { return MA_FALSE; } + if (pFlac->pSeekpoints[iClosestSeekpoint].flacFrameOffset > (ma_uint64)-1 - pFlac->firstFLACFramePosInBytes) { + return MA_FALSE; + } #if !defined(MA_DR_FLAC_NO_CRC) if (pFlac->totalPCMFrameCount > 0) { ma_uint64 byteRangeLo; @@ -88430,16 +89033,26 @@ static ma_bool32 ma_dr_flac__seek_to_pcm_frame__seek_table(ma_dr_flac* pFlac, ma byteRangeLo = pFlac->firstFLACFramePosInBytes + pFlac->pSeekpoints[iClosestSeekpoint].flacFrameOffset; if (iClosestSeekpoint < pFlac->seekpointCount-1) { ma_uint32 iNextSeekpoint = iClosestSeekpoint + 1; - if (pFlac->pSeekpoints[iClosestSeekpoint].flacFrameOffset >= pFlac->pSeekpoints[iNextSeekpoint].flacFrameOffset || pFlac->pSeekpoints[iNextSeekpoint].pcmFrameCount == 0) { - return MA_FALSE; - } if (pFlac->pSeekpoints[iNextSeekpoint].firstPCMFrame != (((ma_uint64)0xFFFFFFFF << 32) | 0xFFFFFFFF)) { + if (pFlac->pSeekpoints[iClosestSeekpoint].flacFrameOffset >= pFlac->pSeekpoints[iNextSeekpoint].flacFrameOffset || pFlac->pSeekpoints[iNextSeekpoint].pcmFrameCount == 0) { + return MA_FALSE; + } + if (pFlac->pSeekpoints[iNextSeekpoint].flacFrameOffset > (ma_uint64)-1 - pFlac->firstFLACFramePosInBytes) { + return MA_FALSE; + } byteRangeHi = pFlac->firstFLACFramePosInBytes + pFlac->pSeekpoints[iNextSeekpoint].flacFrameOffset - 1; } } + if (byteRangeLo > byteRangeHi) { + return MA_FALSE; + } if (ma_dr_flac__seek_to_byte(&pFlac->bs, pFlac->firstFLACFramePosInBytes + pFlac->pSeekpoints[iClosestSeekpoint].flacFrameOffset)) { + MA_DR_FLAC_ZERO_MEMORY(&pFlac->currentFLACFrame, sizeof(pFlac->currentFLACFrame)); if (ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { ma_dr_flac__get_pcm_frame_range_of_current_flac_frame(pFlac, &pFlac->currentPCMFrame, NULL); + if (pFlac->currentPCMFrame > pcmFrameIndex) { + return MA_FALSE; + } if (ma_dr_flac__seek_to_pcm_frame__binary_search_internal(pFlac, pcmFrameIndex, byteRangeLo, byteRangeHi)) { return MA_TRUE; } @@ -88504,7 +89117,7 @@ static ma_bool32 ma_dr_flac__seek_to_pcm_frame__seek_table(ma_dr_flac* pFlac, ma pFlac->currentFLACFrame.pcmFramesRemaining = 0; isMidFrame = MA_FALSE; } - if (pcmFrameIndex == pFlac->totalPCMFrameCount && runningPCMFrameCount == pFlac->totalPCMFrameCount) { + if (runningPCMFrameCount == pcmFrameIndex) { return MA_TRUE; } } @@ -88543,6 +89156,7 @@ typedef struct ma_uint64 totalPCMFrameCount; ma_uint16 maxBlockSizeInPCMFrames; ma_uint64 runningFilePos; + ma_uint64 firstFrameSearchPos; ma_bool32 hasStreamInfoBlock; ma_bool32 hasMetadataBlocks; ma_dr_flac_bs bs; @@ -88645,7 +89259,7 @@ static void* ma_dr_flac__realloc_from_callbacks(void* p, size_t szNew, size_t sz return NULL; } if (p != NULL) { - MA_DR_FLAC_COPY_MEMORY(p2, p, szOld); + MA_DR_FLAC_COPY_MEMORY(p2, p, MA_DR_FLAC_MIN(szNew, szOld)); pAllocationCallbacks->onFree(p, pAllocationCallbacks->pUserData); } return p2; @@ -88661,21 +89275,61 @@ static void ma_dr_flac__free_from_callbacks(void* p, const ma_allocation_callbac pAllocationCallbacks->onFree(p, pAllocationCallbacks->pUserData); } } -static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, ma_dr_flac_meta_proc onMeta, void* pUserData, void* pUserDataMD, ma_uint64* pFirstFramePos, ma_uint64* pSeektablePos, ma_uint32* pSeekpointCount, ma_allocation_callbacks* pAllocationCallbacks) +static ma_bool32 ma_dr_flac__resize_decoded_samples(ma_dr_flac* pFlac, ma_uint16 blockSizeInPCMFrames) { - ma_uint64 runningFilePos = 42; - ma_uint64 seektablePos = 0; - ma_uint32 seektableSize = 0; - (void)onTell; + size_t sampleSize; + size_t allocationSize; + void* pAllocation; + void* pDecodedSamples; + sampleSize = sizeof(ma_int32); + if (pFlac->bitsPerSample == 32) { + sampleSize = sizeof(ma_int64); + } + allocationSize = ma_dr_flac_align(blockSizeInPCMFrames * sampleSize, MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE) * pFlac->channels + MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE; + pAllocation = ma_dr_flac__malloc_from_callbacks(allocationSize, &pFlac->allocationCallbacks); + if (pAllocation == NULL) { + return MA_FALSE; + } + ma_dr_flac__free_from_callbacks(pFlac->pDecodedSamplesAllocation, &pFlac->allocationCallbacks); + pFlac->pDecodedSamplesAllocation = pAllocation; + pFlac->decodedSamplesCapacityInPCMFrames = blockSizeInPCMFrames; + pDecodedSamples = (void*)ma_dr_flac_align((size_t)pAllocation, MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE); + if (pFlac->bitsPerSample != 32) { + pFlac->decodedSamples.pS32 = (ma_int32*)pDecodedSamples; + } else { + pFlac->decodedSamples.pS64 = (ma_int64*)pDecodedSamples; + } + return MA_TRUE; +} +static ma_result ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, ma_dr_flac_meta_proc onMeta, void* pUserData, void* pUserDataMD, ma_uint64* pFirstFramePos, ma_uint64* pSeektablePos, ma_uint32* pSeekpointCount, ma_allocation_callbacks* pAllocationCallbacks) +{ + ma_uint64 runningFilePos = 42; + ma_uint64 seektablePos = 0; + ma_uint32 seektableSize = 0; + ma_int64 fileSize = 0; + ma_bool32 hasKnownFileSize = MA_FALSE; + if (onTell != NULL && onSeek != NULL) { + if (onSeek(pUserData, 0, MA_DR_FLAC_SEEK_END)) { + if (onTell(pUserData, &fileSize)) { + hasKnownFileSize = MA_TRUE; + } + if (!onSeek(pUserData, (int)runningFilePos, MA_DR_FLAC_SEEK_SET)) { + return MA_BAD_SEEK; + } + } + } for (;;) { ma_dr_flac_metadata metadata; ma_uint8 isLastBlock = 0; ma_uint8 blockType = 0; ma_uint32 blockSize; if (ma_dr_flac__read_and_decode_block_header(onRead, pUserData, &isLastBlock, &blockType, &blockSize) == MA_FALSE) { - return MA_FALSE; + return MA_ERROR; } runningFilePos += 4; + if (hasKnownFileSize && (runningFilePos > (ma_uint64)fileSize || blockSize > (ma_uint64)fileSize - runningFilePos)) { + return MA_INVALID_DATA; + } metadata.type = blockType; metadata.rawDataSize = 0; metadata.rawDataOffset = runningFilePos; @@ -88685,16 +89339,16 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea case MA_DR_FLAC_METADATA_BLOCK_TYPE_APPLICATION: { if (blockSize < 4) { - return MA_FALSE; + return MA_INVALID_DATA; } if (onMeta) { void* pRawData = ma_dr_flac__malloc_from_callbacks(blockSize, pAllocationCallbacks); if (pRawData == NULL) { - return MA_FALSE; + return MA_ERROR; } if (onRead(pUserData, pRawData, blockSize) != blockSize) { ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); - return MA_FALSE; + return MA_ERROR; } metadata.pRawData = pRawData; metadata.rawDataSize = blockSize; @@ -88707,29 +89361,34 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea } break; case MA_DR_FLAC_METADATA_BLOCK_TYPE_SEEKTABLE: { + if ((blockSize % MA_DR_FLAC_SEEKPOINT_SIZE_IN_BYTES) != 0) { + return MA_INVALID_DATA; + } seektablePos = runningFilePos; seektableSize = blockSize; if (onMeta) { ma_uint32 seekpointCount; ma_uint32 iSeekpoint; void* pRawData; + size_t rawDataSize; seekpointCount = blockSize/MA_DR_FLAC_SEEKPOINT_SIZE_IN_BYTES; - pRawData = ma_dr_flac__malloc_from_callbacks(seekpointCount * sizeof(ma_dr_flac_seekpoint), pAllocationCallbacks); + rawDataSize = seekpointCount * sizeof(ma_dr_flac_seekpoint); + pRawData = ma_dr_flac__malloc_from_callbacks(rawDataSize, pAllocationCallbacks); if (pRawData == NULL) { - return MA_FALSE; + return MA_ERROR; } for (iSeekpoint = 0; iSeekpoint < seekpointCount; ++iSeekpoint) { ma_dr_flac_seekpoint* pSeekpoint = (ma_dr_flac_seekpoint*)pRawData + iSeekpoint; if (onRead(pUserData, pSeekpoint, MA_DR_FLAC_SEEKPOINT_SIZE_IN_BYTES) != MA_DR_FLAC_SEEKPOINT_SIZE_IN_BYTES) { ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); - return MA_FALSE; + return MA_ERROR; } pSeekpoint->firstPCMFrame = ma_dr_flac__be2host_64(pSeekpoint->firstPCMFrame); pSeekpoint->flacFrameOffset = ma_dr_flac__be2host_64(pSeekpoint->flacFrameOffset); pSeekpoint->pcmFrameCount = ma_dr_flac__be2host_16(pSeekpoint->pcmFrameCount); } metadata.pRawData = pRawData; - metadata.rawDataSize = blockSize; + metadata.rawDataSize = (ma_uint32)rawDataSize; metadata.data.seektable.seekpointCount = seekpointCount; metadata.data.seektable.pSeekpoints = (const ma_dr_flac_seekpoint*)pRawData; onMeta(pUserDataMD, &metadata); @@ -88739,7 +89398,13 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea case MA_DR_FLAC_METADATA_BLOCK_TYPE_VORBIS_COMMENT: { if (blockSize < 8) { - return MA_FALSE; + if (onMeta != NULL && blockSize > 0) { + ma_uint8 unused[8]; + if (onRead(pUserData, unused, blockSize) != blockSize) { + return MA_ERROR; + } + } + break; } if (onMeta) { void* pRawData; @@ -88748,11 +89413,11 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea ma_uint32 i; pRawData = ma_dr_flac__malloc_from_callbacks(blockSize, pAllocationCallbacks); if (pRawData == NULL) { - return MA_FALSE; + return MA_ERROR; } if (onRead(pUserData, pRawData, blockSize) != blockSize) { ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); - return MA_FALSE; + return MA_ERROR; } metadata.pRawData = pRawData; metadata.rawDataSize = blockSize; @@ -88760,37 +89425,34 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea pRunningDataEnd = (const char*)pRawData + blockSize; metadata.data.vorbis_comment.vendorLength = ma_dr_flac__le2host_32_ptr_unaligned(pRunningData); pRunningData += 4; if ((pRunningDataEnd - pRunningData) - 4 < (ma_int64)metadata.data.vorbis_comment.vendorLength) { - ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); - return MA_FALSE; + goto vorbis_comment_done; } - metadata.data.vorbis_comment.vendor = pRunningData; pRunningData += metadata.data.vorbis_comment.vendorLength; + metadata.data.vorbis_comment.vendor = pRunningData; pRunningData += metadata.data.vorbis_comment.vendorLength; metadata.data.vorbis_comment.commentCount = ma_dr_flac__le2host_32_ptr_unaligned(pRunningData); pRunningData += 4; if ((pRunningDataEnd - pRunningData) / sizeof(ma_uint32) < metadata.data.vorbis_comment.commentCount) { - ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); - return MA_FALSE; + goto vorbis_comment_done; } metadata.data.vorbis_comment.pComments = pRunningData; for (i = 0; i < metadata.data.vorbis_comment.commentCount; ++i) { ma_uint32 commentLength; if (pRunningDataEnd - pRunningData < 4) { - ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); - return MA_FALSE; + goto vorbis_comment_done; } commentLength = ma_dr_flac__le2host_32_ptr_unaligned(pRunningData); pRunningData += 4; if (pRunningDataEnd - pRunningData < (ma_int64)commentLength) { - ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); - return MA_FALSE; + goto vorbis_comment_done; } pRunningData += commentLength; } onMeta(pUserDataMD, &metadata); + vorbis_comment_done: ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); } } break; case MA_DR_FLAC_METADATA_BLOCK_TYPE_CUESHEET: { if (blockSize < 396) { - return MA_FALSE; + return MA_INVALID_DATA; } if (onMeta) { void* pRawData; @@ -88802,11 +89464,11 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea void* pTrackData; pRawData = ma_dr_flac__malloc_from_callbacks(blockSize, pAllocationCallbacks); if (pRawData == NULL) { - return MA_FALSE; + return MA_ERROR; } if (onRead(pUserData, pRawData, blockSize) != blockSize) { ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); - return MA_FALSE; + return MA_ERROR; } metadata.pRawData = pRawData; metadata.rawDataSize = blockSize; @@ -88819,13 +89481,13 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea metadata.data.cuesheet.pTrackData = NULL; { const char* pRunningDataSaved = pRunningData; - bufferSize = metadata.data.cuesheet.trackCount * MA_DR_FLAC_CUESHEET_TRACK_SIZE_IN_BYTES; + bufferSize = metadata.data.cuesheet.trackCount * MA_DR_FLAC_CUESHEET_TRACK_PADDED_SIZE_IN_BYTES; for (iTrack = 0; iTrack < metadata.data.cuesheet.trackCount; ++iTrack) { ma_uint8 indexCount; ma_uint32 indexPointSize; if (pRunningDataEnd - pRunningData < MA_DR_FLAC_CUESHEET_TRACK_SIZE_IN_BYTES) { ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); - return MA_FALSE; + return MA_INVALID_DATA; } pRunningData += 35; indexCount = pRunningData[0]; @@ -88834,7 +89496,7 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea indexPointSize = indexCount * MA_DR_FLAC_CUESHEET_TRACK_INDEX_SIZE_IN_BYTES; if (pRunningDataEnd - pRunningData < (ma_int64)indexPointSize) { ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); - return MA_FALSE; + return MA_INVALID_DATA; } pRunningData += indexPointSize; } @@ -88845,7 +89507,7 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea pTrackData = ma_dr_flac__malloc_from_callbacks(bufferSize, pAllocationCallbacks); if (pTrackData == NULL) { ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); - return MA_FALSE; + return MA_ERROR; } pRunningTrackData = (char*)pTrackData; for (iTrack = 0; iTrack < metadata.data.cuesheet.trackCount; ++iTrack) { @@ -88855,7 +89517,7 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea pRunningTrackData += MA_DR_FLAC_CUESHEET_TRACK_SIZE_IN_BYTES-1; indexCount = pRunningData[0]; pRunningData += 1; - pRunningTrackData += 1; + pRunningTrackData += 1 + (MA_DR_FLAC_CUESHEET_TRACK_PADDED_SIZE_IN_BYTES - MA_DR_FLAC_CUESHEET_TRACK_SIZE_IN_BYTES); for (iIndex = 0; iIndex < indexCount; ++iIndex) { ma_dr_flac_cuesheet_track_index* pTrackIndex = (ma_dr_flac_cuesheet_track_index*)pRunningTrackData; MA_DR_FLAC_COPY_MEMORY(pRunningTrackData, pRunningData, MA_DR_FLAC_CUESHEET_TRACK_INDEX_SIZE_IN_BYTES); @@ -88866,9 +89528,9 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea } metadata.data.cuesheet.pTrackData = pTrackData; } + onMeta(pUserDataMD, &metadata); ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); pRawData = NULL; - onMeta(pUserDataMD, &metadata); ma_dr_flac__free_from_callbacks(pTrackData, pAllocationCallbacks); pTrackData = NULL; } @@ -88876,88 +89538,128 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea case MA_DR_FLAC_METADATA_BLOCK_TYPE_PICTURE: { if (blockSize < 32) { - return MA_FALSE; + return MA_INVALID_DATA; } if (onMeta) { - ma_bool32 result = MA_TRUE; + ma_result result = MA_SUCCESS; ma_uint32 blockSizeRemaining = blockSize; char* pMime = NULL; char* pDescription = NULL; void* pPictureData = NULL; - if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.type, 4) != 4) { - result = MA_FALSE; + if (blockSizeRemaining < 4) { + result = MA_INVALID_DATA; + goto done_flac; + } + if (onRead(pUserData, &metadata.data.picture.type, 4) != 4) { + result = MA_ERROR; goto done_flac; } blockSizeRemaining -= 4; metadata.data.picture.type = ma_dr_flac__be2host_32(metadata.data.picture.type); - if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.mimeLength, 4) != 4) { - result = MA_FALSE; + if (blockSizeRemaining < 4) { + result = MA_INVALID_DATA; + goto done_flac; + } + if (onRead(pUserData, &metadata.data.picture.mimeLength, 4) != 4) { + result = MA_ERROR; goto done_flac; } blockSizeRemaining -= 4; metadata.data.picture.mimeLength = ma_dr_flac__be2host_32(metadata.data.picture.mimeLength); + if (blockSizeRemaining < metadata.data.picture.mimeLength) { + result = MA_INVALID_DATA; + goto done_flac; + } pMime = (char*)ma_dr_flac__malloc_from_callbacks(metadata.data.picture.mimeLength + 1, pAllocationCallbacks); if (pMime == NULL) { - result = MA_FALSE; + result = MA_ERROR; goto done_flac; } - if (blockSizeRemaining < metadata.data.picture.mimeLength || onRead(pUserData, pMime, metadata.data.picture.mimeLength) != metadata.data.picture.mimeLength) { - result = MA_FALSE; + if (onRead(pUserData, pMime, metadata.data.picture.mimeLength) != metadata.data.picture.mimeLength) { + result = MA_ERROR; goto done_flac; } blockSizeRemaining -= metadata.data.picture.mimeLength; pMime[metadata.data.picture.mimeLength] = '\0'; metadata.data.picture.mime = (const char*)pMime; - if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.descriptionLength, 4) != 4) { - result = MA_FALSE; + if (blockSizeRemaining < 4) { + result = MA_INVALID_DATA; + goto done_flac; + } + if (onRead(pUserData, &metadata.data.picture.descriptionLength, 4) != 4) { + result = MA_ERROR; goto done_flac; } blockSizeRemaining -= 4; metadata.data.picture.descriptionLength = ma_dr_flac__be2host_32(metadata.data.picture.descriptionLength); + if (blockSizeRemaining < metadata.data.picture.descriptionLength) { + result = MA_INVALID_DATA; + goto done_flac; + } pDescription = (char*)ma_dr_flac__malloc_from_callbacks(metadata.data.picture.descriptionLength + 1, pAllocationCallbacks); if (pDescription == NULL) { - result = MA_FALSE; + result = MA_ERROR; goto done_flac; } - if (blockSizeRemaining < metadata.data.picture.descriptionLength || onRead(pUserData, pDescription, metadata.data.picture.descriptionLength) != metadata.data.picture.descriptionLength) { - result = MA_FALSE; + if (onRead(pUserData, pDescription, metadata.data.picture.descriptionLength) != metadata.data.picture.descriptionLength) { + result = MA_ERROR; goto done_flac; } blockSizeRemaining -= metadata.data.picture.descriptionLength; pDescription[metadata.data.picture.descriptionLength] = '\0'; metadata.data.picture.description = (const char*)pDescription; - if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.width, 4) != 4) { - result = MA_FALSE; + if (blockSizeRemaining < 4) { + result = MA_INVALID_DATA; + goto done_flac; + } + if (onRead(pUserData, &metadata.data.picture.width, 4) != 4) { + result = MA_ERROR; goto done_flac; } blockSizeRemaining -= 4; metadata.data.picture.width = ma_dr_flac__be2host_32(metadata.data.picture.width); - if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.height, 4) != 4) { - result = MA_FALSE; + if (blockSizeRemaining < 4) { + result = MA_INVALID_DATA; + goto done_flac; + } + if (onRead(pUserData, &metadata.data.picture.height, 4) != 4) { + result = MA_ERROR; goto done_flac; } blockSizeRemaining -= 4; metadata.data.picture.height = ma_dr_flac__be2host_32(metadata.data.picture.height); - if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.colorDepth, 4) != 4) { - result = MA_FALSE; + if (blockSizeRemaining < 4) { + result = MA_INVALID_DATA; + goto done_flac; + } + if (onRead(pUserData, &metadata.data.picture.colorDepth, 4) != 4) { + result = MA_ERROR; goto done_flac; } blockSizeRemaining -= 4; metadata.data.picture.colorDepth = ma_dr_flac__be2host_32(metadata.data.picture.colorDepth); - if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.indexColorCount, 4) != 4) { - result = MA_FALSE; + if (blockSizeRemaining < 4) { + result = MA_INVALID_DATA; + goto done_flac; + } + if (onRead(pUserData, &metadata.data.picture.indexColorCount, 4) != 4) { + result = MA_ERROR; goto done_flac; } blockSizeRemaining -= 4; metadata.data.picture.indexColorCount = ma_dr_flac__be2host_32(metadata.data.picture.indexColorCount); - if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.pictureDataSize, 4) != 4) { - result = MA_FALSE; + if (blockSizeRemaining < 4) { + result = MA_INVALID_DATA; + goto done_flac; + } + if (onRead(pUserData, &metadata.data.picture.pictureDataSize, 4) != 4) { + result = MA_ERROR; goto done_flac; } blockSizeRemaining -= 4; metadata.data.picture.pictureDataSize = ma_dr_flac__be2host_32(metadata.data.picture.pictureDataSize); if (blockSizeRemaining < metadata.data.picture.pictureDataSize) { - result = MA_FALSE; + result = MA_INVALID_DATA; goto done_flac; } metadata.data.picture.pictureDataOffset = runningFilePos + (blockSize - blockSizeRemaining); @@ -88965,14 +89667,14 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea pPictureData = ma_dr_flac__malloc_from_callbacks(metadata.data.picture.pictureDataSize, pAllocationCallbacks); if (pPictureData != NULL) { if (onRead(pUserData, pPictureData, metadata.data.picture.pictureDataSize) != metadata.data.picture.pictureDataSize) { - result = MA_FALSE; + result = MA_ERROR; goto done_flac; } } else #endif { if (!onSeek(pUserData, metadata.data.picture.pictureDataSize, MA_DR_FLAC_SEEK_CUR)) { - result = MA_FALSE; + result = MA_ERROR; goto done_flac; } } @@ -88987,8 +89689,8 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea ma_dr_flac__free_from_callbacks(pMime, pAllocationCallbacks); ma_dr_flac__free_from_callbacks(pDescription, pAllocationCallbacks); ma_dr_flac__free_from_callbacks(pPictureData, pAllocationCallbacks); - if (result != MA_TRUE) { - return MA_FALSE; + if (result != MA_SUCCESS) { + return result; } } } break; @@ -88997,7 +89699,7 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea if (onMeta) { metadata.data.padding.unused = 0; if (!onSeek(pUserData, blockSize, MA_DR_FLAC_SEEK_CUR)) { - isLastBlock = MA_TRUE; + return MA_BAD_SEEK; } else { onMeta(pUserDataMD, &metadata); } @@ -89005,11 +89707,7 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea } break; case MA_DR_FLAC_METADATA_BLOCK_TYPE_INVALID: { - if (onMeta) { - if (!onSeek(pUserData, blockSize, MA_DR_FLAC_SEEK_CUR)) { - isLastBlock = MA_TRUE; - } - } + return MA_INVALID_DATA; } break; default: { @@ -89018,11 +89716,11 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea if (pRawData != NULL) { if (onRead(pUserData, pRawData, blockSize) != blockSize) { ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks); - return MA_FALSE; + return MA_ERROR; } } else { if (!onSeek(pUserData, blockSize, MA_DR_FLAC_SEEK_CUR)) { - return MA_FALSE; + return MA_ERROR; } } metadata.pRawData = pRawData; @@ -89034,7 +89732,7 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea } if (onMeta == NULL && blockSize > 0) { if (!onSeek(pUserData, blockSize, MA_DR_FLAC_SEEK_CUR)) { - isLastBlock = MA_TRUE; + return MA_BAD_SEEK; } } runningFilePos += blockSize; @@ -89045,6 +89743,35 @@ static ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRea *pSeektablePos = seektablePos; *pSeekpointCount = seektableSize / MA_DR_FLAC_SEEKPOINT_SIZE_IN_BYTES; *pFirstFramePos = runningFilePos; + return MA_SUCCESS; +} +static ma_bool32 ma_dr_flac__init_private__native_without_streaminfo(ma_dr_flac_init_info* pInit, const ma_uint8* pInitialData, ma_uint64 firstFrameSearchPos) +{ + ma_uint32 initialData; + pInit->container = ma_dr_flac_container_native; + pInit->hasStreamInfoBlock = MA_FALSE; + pInit->hasMetadataBlocks = MA_FALSE; + pInit->firstFrameSearchPos = firstFrameSearchPos; + MA_DR_FLAC_COPY_MEMORY(&initialData, pInitialData, sizeof(initialData)); + initialData = ma_dr_flac__be2host_32(initialData); + pInit->bs.consumedBits = MA_DR_FLAC_CACHE_L1_SIZE_BITS(&pInit->bs) - 32; + pInit->bs.cache = (ma_dr_flac_cache_t)initialData << pInit->bs.consumedBits; + #ifndef MA_DR_FLAC_NO_CRC + { + pInit->bs.crc16Cache = initialData; + pInit->bs.crc16CacheIgnoredBytes = pInit->bs.consumedBits / 8; + } + #endif + if (!ma_dr_flac__read_next_flac_frame_header(&pInit->bs, 0, &pInit->firstFrameHeader)) { + return MA_FALSE; + } + if (pInit->firstFrameHeader.bitsPerSample == 0 || pInit->firstFrameHeader.sampleRate == 0) { + return MA_FALSE; + } + pInit->sampleRate = pInit->firstFrameHeader.sampleRate; + pInit->channels = ma_dr_flac__get_channel_count_from_channel_assignment(pInit->firstFrameHeader.channelAssignment); + pInit->bitsPerSample = pInit->firstFrameHeader.bitsPerSample; + pInit->maxBlockSizeInPCMFrames = 65535; return MA_TRUE; } static ma_bool32 ma_dr_flac__init_private__native(ma_dr_flac_init_info* pInit, ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_meta_proc onMeta, void* pUserData, void* pUserDataMD, ma_bool32 relaxed) @@ -89052,28 +89779,18 @@ static ma_bool32 ma_dr_flac__init_private__native(ma_dr_flac_init_info* pInit, m ma_uint8 isLastBlock; ma_uint8 blockType; ma_uint32 blockSize; + ma_uint32 blockHeader; (void)onSeek; pInit->container = ma_dr_flac_container_native; - if (!ma_dr_flac__read_and_decode_block_header(onRead, pUserData, &isLastBlock, &blockType, &blockSize)) { + if (onRead(pUserData, &blockHeader, 4) != 4) { return MA_FALSE; } + ma_dr_flac__decode_block_header(blockHeader, &isLastBlock, &blockType, &blockSize); if (blockType != MA_DR_FLAC_METADATA_BLOCK_TYPE_STREAMINFO || blockSize != 34) { if (!relaxed) { return MA_FALSE; } else { - pInit->hasStreamInfoBlock = MA_FALSE; - pInit->hasMetadataBlocks = MA_FALSE; - if (!ma_dr_flac__read_next_flac_frame_header(&pInit->bs, 0, &pInit->firstFrameHeader)) { - return MA_FALSE; - } - if (pInit->firstFrameHeader.bitsPerSample == 0) { - return MA_FALSE; - } - pInit->sampleRate = pInit->firstFrameHeader.sampleRate; - pInit->channels = ma_dr_flac__get_channel_count_from_channel_assignment(pInit->firstFrameHeader.channelAssignment); - pInit->bitsPerSample = pInit->firstFrameHeader.bitsPerSample; - pInit->maxBlockSizeInPCMFrames = 65535; - return MA_TRUE; + return ma_dr_flac__init_private__native_without_streaminfo(pInit, (const ma_uint8*)&blockHeader, pInit->runningFilePos); } } else { ma_dr_flac_streaminfo streaminfo; @@ -89522,8 +90239,9 @@ static ma_bool32 ma_dr_flac_ogg__seek_to_pcm_frame(ma_dr_flac* pFlac, ma_uint64 ma_dr_flac_oggbs* oggbs = (ma_dr_flac_oggbs*)pFlac->_oggbs; ma_uint64 originalBytePos; ma_uint64 runningGranulePosition; - ma_uint64 runningFrameBytePos; - ma_uint64 runningPCMFrameCount; + ma_uint64 runningFrameBytePos = 0; + ma_uint64 runningPCMFrameCount = 0; + ma_bool32 hasFramePage = MA_FALSE; MA_DR_FLAC_ASSERT(oggbs != NULL); originalBytePos = oggbs->currentBytePos; if (!ma_dr_flac__seek_to_byte(&pFlac->bs, pFlac->firstFLACFramePosInBytes)) { @@ -89536,21 +90254,24 @@ static ma_bool32 ma_dr_flac_ogg__seek_to_pcm_frame(ma_dr_flac* pFlac, ma_uint64 ma_dr_flac_oggbs__seek_physical(oggbs, originalBytePos, MA_DR_FLAC_SEEK_SET); return MA_FALSE; } - runningFrameBytePos = oggbs->currentBytePos - ma_dr_flac_ogg__get_page_header_size(&oggbs->currentPageHeader) - oggbs->pageDataSize; - if (oggbs->currentPageHeader.granulePosition >= pcmFrameIndex) { - break; - } if ((oggbs->currentPageHeader.headerType & 0x01) == 0) { if (oggbs->currentPageHeader.segmentTable[0] >= 2) { ma_uint8 firstBytesInPage[2]; firstBytesInPage[0] = oggbs->pageData[0]; firstBytesInPage[1] = oggbs->pageData[1]; if ((firstBytesInPage[0] == 0xFF) && (firstBytesInPage[1] & 0xFC) == 0xF8) { - runningGranulePosition = oggbs->currentPageHeader.granulePosition; + runningFrameBytePos = oggbs->currentBytePos - ma_dr_flac_ogg__get_page_header_size(&oggbs->currentPageHeader) - oggbs->pageDataSize; + runningPCMFrameCount = runningGranulePosition; + hasFramePage = MA_TRUE; } - continue; } } + if (oggbs->currentPageHeader.granulePosition != (ma_uint64)-1) { + if (hasFramePage && oggbs->currentPageHeader.granulePosition >= pcmFrameIndex) { + break; + } + runningGranulePosition = oggbs->currentPageHeader.granulePosition; + } } if (!ma_dr_flac_oggbs__seek_physical(oggbs, runningFrameBytePos, MA_DR_FLAC_SEEK_SET)) { return MA_FALSE; @@ -89558,7 +90279,6 @@ static ma_bool32 ma_dr_flac_ogg__seek_to_pcm_frame(ma_dr_flac* pFlac, ma_uint64 if (!ma_dr_flac_oggbs__goto_next_page(oggbs, ma_dr_flac_ogg_recover_on_crc_mismatch)) { return MA_FALSE; } - runningPCMFrameCount = runningGranulePosition; for (;;) { ma_uint64 firstPCMFrameInFLACFrame = 0; ma_uint64 lastPCMFrameInFLACFrame = 0; @@ -89568,7 +90288,7 @@ static ma_bool32 ma_dr_flac_ogg__seek_to_pcm_frame(ma_dr_flac* pFlac, ma_uint64 } ma_dr_flac__get_pcm_frame_range_of_current_flac_frame(pFlac, &firstPCMFrameInFLACFrame, &lastPCMFrameInFLACFrame); pcmFrameCountInThisFrame = (lastPCMFrameInFLACFrame - firstPCMFrameInFLACFrame) + 1; - if (pcmFrameIndex == pFlac->totalPCMFrameCount && (runningPCMFrameCount + pcmFrameCountInThisFrame) == pFlac->totalPCMFrameCount) { + if ((runningPCMFrameCount + pcmFrameCountInThisFrame) == pcmFrameIndex) { ma_result result = ma_dr_flac__decode_flac_frame(pFlac); if (result == MA_SUCCESS) { pFlac->currentPCMFrame = pcmFrameIndex; @@ -89774,7 +90494,7 @@ static ma_bool32 ma_dr_flac__init_private(ma_dr_flac_init_info* pInit, ma_dr_fla #endif if (relaxed) { if (container == ma_dr_flac_container_native) { - return ma_dr_flac__init_private__native(pInit, onRead, onSeek, onMeta, pUserData, pUserDataMD, relaxed); + return ma_dr_flac__init_private__native_without_streaminfo(pInit, id, pInit->runningFilePos - 4); } #ifndef MA_DR_FLAC_NO_OGG if (container == ma_dr_flac_container_ogg) { @@ -89793,6 +90513,7 @@ static void ma_dr_flac__init_from_info(ma_dr_flac* pFlac, const ma_dr_flac_init_ pFlac->onMeta = pInit->onMeta; pFlac->pUserDataMD = pInit->pUserDataMD; pFlac->maxBlockSizeInPCMFrames = pInit->maxBlockSizeInPCMFrames; + pFlac->decodedSamplesCapacityInPCMFrames = pInit->maxBlockSizeInPCMFrames; pFlac->sampleRate = pInit->sampleRate; pFlac->channels = (ma_uint8)pInit->channels; pFlac->bitsPerSample = (ma_uint8)pInit->bitsPerSample; @@ -89803,14 +90524,17 @@ static ma_dr_flac* ma_dr_flac_open_with_metadata_private(ma_dr_flac_read_proc on { ma_dr_flac_init_info init; ma_uint32 allocationSize; + ma_uint32 decodedSampleSize; ma_uint32 wholeSIMDVectorCountPerChannel; ma_uint32 decodedSamplesAllocationSize; + ma_uint8* pDecodedSamples; #ifndef MA_DR_FLAC_NO_OGG ma_dr_flac_oggbs* pOggbs = NULL; #endif ma_uint64 firstFramePos; ma_uint64 seektablePos; ma_uint32 seekpointCount; + ma_bool32 recoverMetadata = MA_FALSE; ma_allocation_callbacks allocationCallbacks; ma_dr_flac* pFlac; ma_dr_flac__init_cpu_caps(); @@ -89829,10 +90553,14 @@ static ma_dr_flac* ma_dr_flac_open_with_metadata_private(ma_dr_flac_read_proc on allocationCallbacks.onFree = ma_dr_flac__free_default; } allocationSize = sizeof(ma_dr_flac); - if ((init.maxBlockSizeInPCMFrames % (MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE / sizeof(ma_int32))) == 0) { - wholeSIMDVectorCountPerChannel = (init.maxBlockSizeInPCMFrames / (MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE / sizeof(ma_int32))); + decodedSampleSize = sizeof(ma_int32); + if (init.bitsPerSample == 32) { + decodedSampleSize = sizeof(ma_int64); + } + if ((init.maxBlockSizeInPCMFrames % (MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE / decodedSampleSize)) == 0) { + wholeSIMDVectorCountPerChannel = (init.maxBlockSizeInPCMFrames / (MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE / decodedSampleSize)); } else { - wholeSIMDVectorCountPerChannel = (init.maxBlockSizeInPCMFrames / (MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE / sizeof(ma_int32))) + 1; + wholeSIMDVectorCountPerChannel = (init.maxBlockSizeInPCMFrames / (MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE / decodedSampleSize)) + 1; } decodedSamplesAllocationSize = wholeSIMDVectorCountPerChannel * MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE * init.channels; allocationSize += decodedSamplesAllocationSize; @@ -89857,9 +90585,13 @@ static ma_dr_flac* ma_dr_flac_open_with_metadata_private(ma_dr_flac_read_proc on } #endif firstFramePos = 42; + if (!init.hasStreamInfoBlock) { + firstFramePos = init.firstFrameSearchPos; + } seektablePos = 0; seekpointCount = 0; if (init.hasMetadataBlocks) { + ma_result metadataResult; ma_dr_flac_read_proc onReadOverride = onRead; ma_dr_flac_seek_proc onSeekOverride = onSeek; ma_dr_flac_tell_proc onTellOverride = onTell; @@ -89872,7 +90604,21 @@ static ma_dr_flac* ma_dr_flac_open_with_metadata_private(ma_dr_flac_read_proc on pUserDataOverride = (void*)pOggbs; } #endif - if (!ma_dr_flac__read_and_decode_metadata(onReadOverride, onSeekOverride, onTellOverride, onMeta, pUserDataOverride, pUserDataMD, &firstFramePos, &seektablePos, &seekpointCount, &allocationCallbacks)) { + metadataResult = ma_dr_flac__read_and_decode_metadata(onReadOverride, onSeekOverride, onTellOverride, onMeta, pUserDataOverride, pUserDataMD, &firstFramePos, &seektablePos, &seekpointCount, &allocationCallbacks); + if (metadataResult != MA_SUCCESS) { + if (metadataResult == MA_INVALID_DATA && init.container == ma_dr_flac_container_native && onSeek != NULL) { + firstFramePos = 42; + seektablePos = 0; + seekpointCount = 0; + recoverMetadata = MA_TRUE; + } else { + #ifndef MA_DR_FLAC_NO_OGG + ma_dr_flac__free_from_callbacks(pOggbs, &allocationCallbacks); + #endif + return NULL; + } + } + if ((0xFFFFFFFF - (seekpointCount * sizeof(ma_dr_flac_seekpoint))) < allocationSize) { #ifndef MA_DR_FLAC_NO_OGG ma_dr_flac__free_from_callbacks(pOggbs, &allocationCallbacks); #endif @@ -89880,7 +90626,7 @@ static ma_dr_flac* ma_dr_flac_open_with_metadata_private(ma_dr_flac_read_proc on } allocationSize += seekpointCount * sizeof(ma_dr_flac_seekpoint); } - pFlac = (ma_dr_flac*)ma_dr_flac__malloc_from_callbacks(allocationSize, &allocationCallbacks); + pFlac = (ma_dr_flac*)ma_dr_flac__malloc_from_callbacks((size_t)allocationSize, &allocationCallbacks); if (pFlac == NULL) { #ifndef MA_DR_FLAC_NO_OGG ma_dr_flac__free_from_callbacks(pOggbs, &allocationCallbacks); @@ -89889,10 +90635,15 @@ static ma_dr_flac* ma_dr_flac_open_with_metadata_private(ma_dr_flac_read_proc on } ma_dr_flac__init_from_info(pFlac, &init); pFlac->allocationCallbacks = allocationCallbacks; - pFlac->pDecodedSamples = (ma_int32*)ma_dr_flac_align((size_t)pFlac->pExtraData, MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE); + pDecodedSamples = (ma_uint8*)ma_dr_flac_align((size_t)pFlac->pExtraData, MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE); + if (init.bitsPerSample != 32) { + pFlac->decodedSamples.pS32 = (ma_int32*)pDecodedSamples; + } else { + pFlac->decodedSamples.pS64 = (ma_int64*)pDecodedSamples; + } #ifndef MA_DR_FLAC_NO_OGG if (init.container == ma_dr_flac_container_ogg) { - ma_dr_flac_oggbs* pInternalOggbs = (ma_dr_flac_oggbs*)((ma_uint8*)pFlac->pDecodedSamples + decodedSamplesAllocationSize + (seekpointCount * sizeof(ma_dr_flac_seekpoint))); + ma_dr_flac_oggbs* pInternalOggbs = (ma_dr_flac_oggbs*)(pDecodedSamples + decodedSamplesAllocationSize + (seekpointCount * sizeof(ma_dr_flac_seekpoint))); MA_DR_FLAC_COPY_MEMORY(pInternalOggbs, pOggbs, sizeof(*pOggbs)); ma_dr_flac__free_from_callbacks(pOggbs, &allocationCallbacks); pOggbs = NULL; @@ -89915,7 +90666,7 @@ static ma_dr_flac* ma_dr_flac_open_with_metadata_private(ma_dr_flac_read_proc on { if (seektablePos != 0) { pFlac->seekpointCount = seekpointCount; - pFlac->pSeekpoints = (ma_dr_flac_seekpoint*)((ma_uint8*)pFlac->pDecodedSamples + decodedSamplesAllocationSize); + pFlac->pSeekpoints = (ma_dr_flac_seekpoint*)(pDecodedSamples + decodedSamplesAllocationSize); MA_DR_FLAC_ASSERT(pFlac->bs.onSeek != NULL); MA_DR_FLAC_ASSERT(pFlac->bs.onRead != NULL); if (pFlac->bs.onSeek(pFlac->bs.pUserData, (int)seektablePos, MA_DR_FLAC_SEEK_SET)) { @@ -89932,6 +90683,7 @@ static ma_dr_flac* ma_dr_flac_open_with_metadata_private(ma_dr_flac_read_proc on } } if (!pFlac->bs.onSeek(pFlac->bs.pUserData, (int)pFlac->firstFLACFramePosInBytes, MA_DR_FLAC_SEEK_SET)) { + ma_dr_flac__free_from_callbacks(pFlac->pDecodedSamplesAllocation, &allocationCallbacks); ma_dr_flac__free_from_callbacks(pFlac, &allocationCallbacks); return NULL; } @@ -89941,7 +90693,13 @@ static ma_dr_flac* ma_dr_flac_open_with_metadata_private(ma_dr_flac_read_proc on } } } - if (!init.hasStreamInfoBlock) { + if (recoverMetadata) { + if (!ma_dr_flac__seek_to_byte(&pFlac->bs, firstFramePos) || !ma_dr_flac__read_and_decode_next_flac_frame(pFlac)) { + ma_dr_flac__free_from_callbacks(pFlac->pDecodedSamplesAllocation, &allocationCallbacks); + ma_dr_flac__free_from_callbacks(pFlac, &allocationCallbacks); + return NULL; + } + } else if (!init.hasStreamInfoBlock) { pFlac->currentFLACFrame.header = init.firstFrameHeader; for (;;) { ma_result result = ma_dr_flac__decode_flac_frame(pFlac); @@ -89950,11 +90708,13 @@ static ma_dr_flac* ma_dr_flac_open_with_metadata_private(ma_dr_flac_read_proc on } else { if (result == MA_CRC_MISMATCH) { if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) { + ma_dr_flac__free_from_callbacks(pFlac->pDecodedSamplesAllocation, &allocationCallbacks); ma_dr_flac__free_from_callbacks(pFlac, &allocationCallbacks); return NULL; } continue; } else { + ma_dr_flac__free_from_callbacks(pFlac->pDecodedSamplesAllocation, &allocationCallbacks); ma_dr_flac__free_from_callbacks(pFlac, &allocationCallbacks); return NULL; } @@ -90195,6 +90955,7 @@ MA_API void ma_dr_flac_close(ma_dr_flac* pFlac) } #endif #endif + ma_dr_flac__free_from_callbacks(pFlac->pDecodedSamplesAllocation, &pFlac->allocationCallbacks); ma_dr_flac__free_from_callbacks(pFlac, &pFlac->allocationCallbacks); } #if 0 @@ -90257,7 +91018,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_left_side__sse2(ma_ const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1; ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); for (i = 0; i < frameCount4; ++i) { __m128i left = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); __m128i side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1); @@ -90285,7 +91046,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_left_side__neon(ma_ ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; int32x4_t shift0_4; int32x4_t shift1_4; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); shift0_4 = vdupq_n_s32(shift0); shift1_4 = vdupq_n_s32(shift1); for (i = 0; i < frameCount4; ++i) { @@ -90306,14 +91067,30 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_left_side__neon(ma_ } } #endif +static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_duplicate_mono(ma_uint64 frameCount, ma_uint32 shift, const ma_int32* pInputSamples, ma_int32* pOutputSamples) +{ + ma_uint64 i; + const ma_uint32* pInputSamplesU32 = (const ma_uint32*)pInputSamples; + MA_DR_FLAC_ASSERT(shift < 32); + for (i = 0; i < frameCount; ++i) { + ma_int32 sample; + sample = (ma_int32)(pInputSamplesU32[i] << shift); + pOutputSamples[i*2+0] = sample; + pOutputSamples[i*2+1] = sample; + } +} static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_left_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples) { + if (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample == 32) { + ma_dr_flac_read_pcm_frames_s32__decode_duplicate_mono(frameCount, unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample, pInputSamples0, pOutputSamples); + return; + } #if defined(MA_DR_FLAC_SUPPORT_SSE2) - if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsSSE2Supported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s32__decode_left_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #elif defined(MA_DR_FLAC_SUPPORT_NEON) - if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsNEONSupported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s32__decode_left_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #endif @@ -90385,7 +91162,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_right_side__sse2(ma const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1; ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); for (i = 0; i < frameCount4; ++i) { __m128i side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); __m128i right = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1); @@ -90413,7 +91190,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_right_side__neon(ma ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; int32x4_t shift0_4; int32x4_t shift1_4; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); shift0_4 = vdupq_n_s32(shift0); shift1_4 = vdupq_n_s32(shift1); for (i = 0; i < frameCount4; ++i) { @@ -90436,12 +91213,16 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_right_side__neon(ma #endif static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_right_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples) { + if (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample == 32) { + ma_dr_flac_read_pcm_frames_s32__decode_duplicate_mono(frameCount, unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample, pInputSamples1, pOutputSamples); + return; + } #if defined(MA_DR_FLAC_SUPPORT_SSE2) - if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsSSE2Supported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s32__decode_right_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #elif defined(MA_DR_FLAC_SUPPORT_NEON) - if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsNEONSupported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s32__decode_right_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #endif @@ -90473,7 +91254,6 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_mid_side__scalar(ma const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1; ma_int32 shift = unusedBitsPerSample; if (shift > 0) { - shift -= 1; for (i = 0; i < frameCount4; ++i) { ma_uint32 temp0L; ma_uint32 temp1L; @@ -90495,14 +91275,14 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_mid_side__scalar(ma mid1 = (mid1 << 1) | (side1 & 0x01); mid2 = (mid2 << 1) | (side2 & 0x01); mid3 = (mid3 << 1) | (side3 & 0x01); - temp0L = (mid0 + side0) << shift; - temp1L = (mid1 + side1) << shift; - temp2L = (mid2 + side2) << shift; - temp3L = (mid3 + side3) << shift; - temp0R = (mid0 - side0) << shift; - temp1R = (mid1 - side1) << shift; - temp2R = (mid2 - side2) << shift; - temp3R = (mid3 - side3) << shift; + temp0L = (mid0 + side0) << (shift - 1); + temp1L = (mid1 + side1) << (shift - 1); + temp2L = (mid2 + side2) << (shift - 1); + temp3L = (mid3 + side3) << (shift - 1); + temp0R = (mid0 - side0) << (shift - 1); + temp1R = (mid1 - side1) << (shift - 1); + temp2R = (mid2 - side2) << (shift - 1); + temp3R = (mid3 - side3) << (shift - 1); pOutputSamples[i*8+0] = (ma_int32)temp0L; pOutputSamples[i*8+1] = (ma_int32)temp0R; pOutputSamples[i*8+2] = (ma_int32)temp1L; @@ -90522,26 +91302,26 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_mid_side__scalar(ma ma_uint32 temp1R; ma_uint32 temp2R; ma_uint32 temp3R; - ma_uint32 mid0 = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 mid1 = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 mid2 = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 mid3 = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - ma_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - ma_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - ma_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + ma_uint64 mid0 = (ma_uint64)(ma_int64)pInputSamples0[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint64 mid1 = (ma_uint64)(ma_int64)pInputSamples0[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint64 mid2 = (ma_uint64)(ma_int64)pInputSamples0[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint64 mid3 = (ma_uint64)(ma_int64)pInputSamples0[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint64 side0 = (ma_uint64)(ma_int64)pInputSamples1[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + ma_uint64 side1 = (ma_uint64)(ma_int64)pInputSamples1[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + ma_uint64 side2 = (ma_uint64)(ma_int64)pInputSamples1[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + ma_uint64 side3 = (ma_uint64)(ma_int64)pInputSamples1[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; mid0 = (mid0 << 1) | (side0 & 0x01); mid1 = (mid1 << 1) | (side1 & 0x01); mid2 = (mid2 << 1) | (side2 & 0x01); mid3 = (mid3 << 1) | (side3 & 0x01); - temp0L = (ma_uint32)((ma_int32)(mid0 + side0) >> 1); - temp1L = (ma_uint32)((ma_int32)(mid1 + side1) >> 1); - temp2L = (ma_uint32)((ma_int32)(mid2 + side2) >> 1); - temp3L = (ma_uint32)((ma_int32)(mid3 + side3) >> 1); - temp0R = (ma_uint32)((ma_int32)(mid0 - side0) >> 1); - temp1R = (ma_uint32)((ma_int32)(mid1 - side1) >> 1); - temp2R = (ma_uint32)((ma_int32)(mid2 - side2) >> 1); - temp3R = (ma_uint32)((ma_int32)(mid3 - side3) >> 1); + temp0L = (ma_uint32)((ma_int64)(mid0 + side0) >> 1); + temp1L = (ma_uint32)((ma_int64)(mid1 + side1) >> 1); + temp2L = (ma_uint32)((ma_int64)(mid2 + side2) >> 1); + temp3L = (ma_uint32)((ma_int64)(mid3 + side3) >> 1); + temp0R = (ma_uint32)((ma_int64)(mid0 - side0) >> 1); + temp1R = (ma_uint32)((ma_int64)(mid1 - side1) >> 1); + temp2R = (ma_uint32)((ma_int64)(mid2 - side2) >> 1); + temp3R = (ma_uint32)((ma_int64)(mid3 - side3) >> 1); pOutputSamples[i*8+0] = (ma_int32)temp0L; pOutputSamples[i*8+1] = (ma_int32)temp0R; pOutputSamples[i*8+2] = (ma_int32)temp1L; @@ -90552,12 +91332,24 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_mid_side__scalar(ma pOutputSamples[i*8+7] = (ma_int32)temp3R; } } - for (i = (frameCount4 << 2); i < frameCount; ++i) { - ma_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - mid = (mid << 1) | (side & 0x01); - pOutputSamples[i*2+0] = (ma_int32)((ma_uint32)((ma_int32)(mid + side) >> 1) << unusedBitsPerSample); - pOutputSamples[i*2+1] = (ma_int32)((ma_uint32)((ma_int32)(mid - side) >> 1) << unusedBitsPerSample); + if (shift > 0) { + for (i = (frameCount4 << 2); i < frameCount; ++i) { + ma_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + mid = (mid << 1) | (side & 0x01); + pOutputSamples[i*2+0] = (ma_int32)((ma_uint32)((ma_int32)(mid + side) >> 1) << unusedBitsPerSample); + pOutputSamples[i*2+1] = (ma_int32)((ma_uint32)((ma_int32)(mid - side) >> 1) << unusedBitsPerSample); + } + } else { + for (i = (frameCount4 << 2); i < frameCount; ++i) { + ma_uint64 mid; + ma_uint64 side; + mid = (ma_uint64)(ma_int64)pInputSamples0[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + side = (ma_uint64)(ma_int64)pInputSamples1[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + mid = (mid << 1) | (side & 0x01); + pOutputSamples[i*2+0] = (ma_int32)((ma_int64)(mid + side) >> 1); + pOutputSamples[i*2+1] = (ma_int32)((ma_int64)(mid - side) >> 1); + } } } #if defined(MA_DR_FLAC_SUPPORT_SSE2) @@ -90568,7 +91360,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_mid_side__sse2(ma_d const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0; const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1; ma_int32 shift = unusedBitsPerSample; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); if (shift == 0) { for (i = 0; i < frameCount4; ++i) { __m128i mid; @@ -90626,7 +91418,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_mid_side__neon(ma_d int32x4_t wbpsShift0_4; int32x4_t wbpsShift1_4; uint32x4_t one4; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); wbpsShift0_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); wbpsShift1_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); one4 = vdupq_n_u32(1); @@ -90679,11 +91471,11 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_mid_side__neon(ma_d static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_mid_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples) { #if defined(MA_DR_FLAC_SUPPORT_SSE2) - if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsSSE2Supported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s32__decode_mid_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #elif defined(MA_DR_FLAC_SUPPORT_NEON) - if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsNEONSupported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s32__decode_mid_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #endif @@ -90783,11 +91575,11 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo_ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples) { #if defined(MA_DR_FLAC_SUPPORT_SSE2) - if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsSSE2Supported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #elif defined(MA_DR_FLAC_SUPPORT_NEON) - if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsNEONSupported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #endif @@ -90799,10 +91591,181 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo( #endif } } +static MA_INLINE void ma_dr_flac__decode_stereo_sample_s64(const ma_dr_flac_frame* pFrame, ma_uint64 iPCMFrame, ma_int32* pLeft, ma_int32* pRight) +{ + ma_uint64 sample0; + ma_uint64 sample1; + sample0 = (ma_uint64)pFrame->subframes[0].samples.pS64[iPCMFrame] << pFrame->subframes[0].wastedBitsPerSample; + sample1 = (ma_uint64)pFrame->subframes[1].samples.pS64[iPCMFrame] << pFrame->subframes[1].wastedBitsPerSample; + switch (pFrame->header.channelAssignment) + { + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE: + { + *pLeft = (ma_int32)sample0; + *pRight = (ma_int32)(sample0 - sample1); + } break; + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE: + { + *pLeft = (ma_int32)(sample0 + sample1); + *pRight = (ma_int32)sample1; + } break; + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE: + { + sample0 = (sample0 << 1) | (sample1 & 0x01); + *pLeft = (ma_int32)((ma_int64)(sample0 + sample1) >> 1); + *pRight = (ma_int32)((ma_int64)(sample0 - sample1) >> 1); + } break; + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_INDEPENDENT_STEREO: + { + *pLeft = (ma_int32)sample0; + *pRight = (ma_int32)sample1; + } break; + default: + { + MA_DR_FLAC_ASSERT(!"Invalid stereo channel assignment"); + *pLeft = 0; + *pRight = 0; + } break; + } +} +static void ma_dr_flac_read_pcm_frames_s32__decode_s64(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint64 iFirstPCMFrame, ma_int32* pOutputSamples) +{ + ma_uint64 i; + MA_DR_FLAC_ASSERT(pFlac->bitsPerSample == 32); + if (pFlac->channels == 2) { + for (i = 0; i < frameCount; ++i) { + ma_int32 left; + ma_int32 right; + ma_dr_flac__decode_stereo_sample_s64(&pFlac->currentFLACFrame, iFirstPCMFrame + i, &left, &right); + pOutputSamples[i*2+0] = left; + pOutputSamples[i*2+1] = right; + } + } else { + for (i = 0; i < frameCount; ++i) { + unsigned int j; + for (j = 0; j < pFlac->channels; ++j) { + ma_int32 sample; + sample = (ma_int32)((ma_uint64)pFlac->currentFLACFrame.subframes[j].samples.pS64[iFirstPCMFrame + i] << pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample); + pOutputSamples[i*pFlac->channels+j] = sample; + } + } + } +} +static void ma_dr_flac_read_pcm_frames_s16__decode_s64(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint64 iFirstPCMFrame, ma_int16* pOutputSamples) +{ + ma_uint64 i; + MA_DR_FLAC_ASSERT(pFlac->bitsPerSample == 32); + if (pFlac->channels == 2) { + for (i = 0; i < frameCount; ++i) { + ma_int32 left; + ma_int32 right; + ma_dr_flac__decode_stereo_sample_s64(&pFlac->currentFLACFrame, iFirstPCMFrame + i, &left, &right); + pOutputSamples[i*2+0] = (ma_int16)(left >> 16); + pOutputSamples[i*2+1] = (ma_int16)(right >> 16); + } + } else { + for (i = 0; i < frameCount; ++i) { + unsigned int j; + for (j = 0; j < pFlac->channels; ++j) { + ma_int32 sample; + sample = (ma_int32)((ma_uint64)pFlac->currentFLACFrame.subframes[j].samples.pS64[iFirstPCMFrame + i] << pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample); + pOutputSamples[i*pFlac->channels+j] = (ma_int16)(sample >> 16); + } + } + } +} +static void ma_dr_flac_read_pcm_frames_f32__decode_s64(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint64 iFirstPCMFrame, float* pOutputSamples) +{ + ma_uint64 i; + MA_DR_FLAC_ASSERT(pFlac->bitsPerSample == 32); + if (pFlac->channels == 2) { + for (i = 0; i < frameCount; ++i) { + ma_int32 left; + ma_int32 right; + ma_dr_flac__decode_stereo_sample_s64(&pFlac->currentFLACFrame, iFirstPCMFrame + i, &left, &right); + pOutputSamples[i*2+0] = left * (1.0f / 2147483648.0f); + pOutputSamples[i*2+1] = right * (1.0f / 2147483648.0f); + } + } else { + for (i = 0; i < frameCount; ++i) { + unsigned int j; + for (j = 0; j < pFlac->channels; ++j) { + ma_int32 sample; + sample = (ma_int32)((ma_uint64)pFlac->currentFLACFrame.subframes[j].samples.pS64[iFirstPCMFrame + i] << pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample); + pOutputSamples[i*pFlac->channels+j] = sample * (1.0f / 2147483648.0f); + } + } + } +} +static ma_bool32 ma_dr_flac__is_current_flac_frame_valid(ma_dr_flac* pFlac) +{ + ma_uint32 iChannel; + ma_uint32 channelCount; + if (pFlac->currentFLACFrame.header.blockSizeInPCMFrames > pFlac->decodedSamplesCapacityInPCMFrames || pFlac->currentFLACFrame.pcmFramesRemaining > pFlac->currentFLACFrame.header.blockSizeInPCMFrames) { + return MA_FALSE; + } + channelCount = ma_dr_flac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment); + if (channelCount != pFlac->channels) { + return MA_FALSE; + } + if (pFlac->bitsPerSample == 32) { + for (iChannel = 0; iChannel < channelCount; iChannel += 1) { + if (pFlac->currentFLACFrame.subframes[iChannel].samples.pS64 == NULL) { + return MA_FALSE; + } + } + } else { + for (iChannel = 0; iChannel < channelCount; iChannel += 1) { + if (pFlac->currentFLACFrame.subframes[iChannel].samples.pS32 == NULL) { + return MA_FALSE; + } + } + } + return MA_TRUE; +} +static MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint64 iFirstPCMFrame, ma_int32* pOutputSamples) +{ + unsigned int channelCount; + ma_uint32 unusedBitsPerSample; + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample < 32); + channelCount = ma_dr_flac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment); + unusedBitsPerSample = 32 - pFlac->currentFLACFrame.header.bitsPerSample; + if (channelCount == 2) { + const ma_int32* pDecodedSamples0 = pFlac->currentFLACFrame.subframes[0].samples.pS32 + iFirstPCMFrame; + const ma_int32* pDecodedSamples1 = pFlac->currentFLACFrame.subframes[1].samples.pS32 + iFirstPCMFrame; + switch (pFlac->currentFLACFrame.header.channelAssignment) + { + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE: + { + ma_dr_flac_read_pcm_frames_s32__decode_left_side(pFlac, frameCount, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pOutputSamples); + } break; + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE: + { + ma_dr_flac_read_pcm_frames_s32__decode_right_side(pFlac, frameCount, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pOutputSamples); + } break; + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE: + { + ma_dr_flac_read_pcm_frames_s32__decode_mid_side(pFlac, frameCount, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pOutputSamples); + } break; + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_INDEPENDENT_STEREO: + { + ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo(pFlac, frameCount, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pOutputSamples); + } break; + default: MA_DR_FLAC_ASSERT(!"Invalid stereo channel assignment"); break; + } + } else { + ma_uint64 i; + for (i = 0; i < frameCount; ++i) { + unsigned int j; + for (j = 0; j < channelCount; ++j) { + pOutputSamples[(i*channelCount)+j] = (ma_int32)((ma_uint32)(pFlac->currentFLACFrame.subframes[j].samples.pS32[iFirstPCMFrame + i]) << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample)); + } + } + } +} MA_API ma_uint64 ma_dr_flac_read_pcm_frames_s32(ma_dr_flac* pFlac, ma_uint64 framesToRead, ma_int32* pBufferOut) { ma_uint64 framesRead; - ma_uint32 unusedBitsPerSample; if (pFlac == NULL || framesToRead == 0) { return 0; } @@ -90810,7 +91773,6 @@ MA_API ma_uint64 ma_dr_flac_read_pcm_frames_s32(ma_dr_flac* pFlac, ma_uint64 fra return ma_dr_flac__seek_forward_by_pcm_frames(pFlac, framesToRead); } MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 32); - unusedBitsPerSample = 32 - pFlac->bitsPerSample; framesRead = 0; while (framesToRead > 0) { if (pFlac->currentFLACFrame.pcmFramesRemaining == 0) { @@ -90824,42 +91786,15 @@ MA_API ma_uint64 ma_dr_flac_read_pcm_frames_s32(ma_dr_flac* pFlac, ma_uint64 fra if (frameCountThisIteration > pFlac->currentFLACFrame.pcmFramesRemaining) { frameCountThisIteration = pFlac->currentFLACFrame.pcmFramesRemaining; } - if (channelCount == 2) { - const ma_int32* pDecodedSamples0 = pFlac->currentFLACFrame.subframes[0].pSamplesS32 + iFirstPCMFrame; - const ma_int32* pDecodedSamples1 = pFlac->currentFLACFrame.subframes[1].pSamplesS32 + iFirstPCMFrame; - switch (pFlac->currentFLACFrame.header.channelAssignment) - { - case MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE: - { - ma_dr_flac_read_pcm_frames_s32__decode_left_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); - } break; - case MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE: - { - ma_dr_flac_read_pcm_frames_s32__decode_right_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); - } break; - case MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE: - { - ma_dr_flac_read_pcm_frames_s32__decode_mid_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); - } break; - case MA_DR_FLAC_CHANNEL_ASSIGNMENT_INDEPENDENT: - default: - { - ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); - } break; - } + if (pFlac->bitsPerSample != 32) { + ma_dr_flac_read_pcm_frames_s32__decode(pFlac, frameCountThisIteration, iFirstPCMFrame, pBufferOut); } else { - ma_uint64 i; - for (i = 0; i < frameCountThisIteration; ++i) { - unsigned int j; - for (j = 0; j < channelCount; ++j) { - pBufferOut[(i*channelCount)+j] = (ma_int32)((ma_uint32)(pFlac->currentFLACFrame.subframes[j].pSamplesS32[iFirstPCMFrame + i]) << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample)); - } - } + ma_dr_flac_read_pcm_frames_s32__decode_s64(pFlac, frameCountThisIteration, iFirstPCMFrame, pBufferOut); } - framesRead += frameCountThisIteration; - pBufferOut += frameCountThisIteration * channelCount; - framesToRead -= frameCountThisIteration; - pFlac->currentPCMFrame += frameCountThisIteration; + framesRead += frameCountThisIteration; + pBufferOut += frameCountThisIteration * channelCount; + framesToRead -= frameCountThisIteration; + pFlac->currentPCMFrame += frameCountThisIteration; pFlac->currentFLACFrame.pcmFramesRemaining -= (ma_uint32)frameCountThisIteration; } } @@ -90937,7 +91872,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_left_side__sse2(ma_ const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1; ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); for (i = 0; i < frameCount4; ++i) { __m128i left = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); __m128i side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1); @@ -90968,7 +91903,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_left_side__neon(ma_ ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; int32x4_t shift0_4; int32x4_t shift1_4; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); shift0_4 = vdupq_n_s32(shift0); shift1_4 = vdupq_n_s32(shift1); for (i = 0; i < frameCount4; ++i) { @@ -90993,14 +91928,30 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_left_side__neon(ma_ } } #endif +static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_duplicate_mono(ma_uint64 frameCount, ma_uint32 shift, const ma_int32* pInputSamples, ma_int16* pOutputSamples) +{ + ma_uint64 i; + const ma_uint32* pInputSamplesU32 = (const ma_uint32*)pInputSamples; + MA_DR_FLAC_ASSERT(shift < 32); + for (i = 0; i < frameCount; ++i) { + ma_int16 sample; + sample = (ma_int16)((pInputSamplesU32[i] << shift) >> 16); + pOutputSamples[i*2+0] = sample; + pOutputSamples[i*2+1] = sample; + } +} static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_left_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples) { + if (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample == 32) { + ma_dr_flac_read_pcm_frames_s16__decode_duplicate_mono(frameCount, unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample, pInputSamples0, pOutputSamples); + return; + } #if defined(MA_DR_FLAC_SUPPORT_SSE2) - if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsSSE2Supported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s16__decode_left_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #elif defined(MA_DR_FLAC_SUPPORT_NEON) - if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsNEONSupported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s16__decode_left_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #endif @@ -91084,7 +92035,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_right_side__sse2(ma const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1; ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); for (i = 0; i < frameCount4; ++i) { __m128i side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); __m128i right = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1); @@ -91115,7 +92066,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_right_side__neon(ma ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; int32x4_t shift0_4; int32x4_t shift1_4; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); shift0_4 = vdupq_n_s32(shift0); shift1_4 = vdupq_n_s32(shift1); for (i = 0; i < frameCount4; ++i) { @@ -91142,12 +92093,16 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_right_side__neon(ma #endif static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_right_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples) { + if (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample == 32) { + ma_dr_flac_read_pcm_frames_s16__decode_duplicate_mono(frameCount, unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample, pInputSamples1, pOutputSamples); + return; + } #if defined(MA_DR_FLAC_SUPPORT_SSE2) - if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsSSE2Supported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s16__decode_right_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #elif defined(MA_DR_FLAC_SUPPORT_NEON) - if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsNEONSupported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s16__decode_right_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #endif @@ -91179,7 +92134,6 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_mid_side__scalar(ma const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1; ma_uint32 shift = unusedBitsPerSample; if (shift > 0) { - shift -= 1; for (i = 0; i < frameCount4; ++i) { ma_uint32 temp0L; ma_uint32 temp1L; @@ -91201,14 +92155,14 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_mid_side__scalar(ma mid1 = (mid1 << 1) | (side1 & 0x01); mid2 = (mid2 << 1) | (side2 & 0x01); mid3 = (mid3 << 1) | (side3 & 0x01); - temp0L = (mid0 + side0) << shift; - temp1L = (mid1 + side1) << shift; - temp2L = (mid2 + side2) << shift; - temp3L = (mid3 + side3) << shift; - temp0R = (mid0 - side0) << shift; - temp1R = (mid1 - side1) << shift; - temp2R = (mid2 - side2) << shift; - temp3R = (mid3 - side3) << shift; + temp0L = (mid0 + side0) << (shift - 1); + temp1L = (mid1 + side1) << (shift - 1); + temp2L = (mid2 + side2) << (shift - 1); + temp3L = (mid3 + side3) << (shift - 1); + temp0R = (mid0 - side0) << (shift - 1); + temp1R = (mid1 - side1) << (shift - 1); + temp2R = (mid2 - side2) << (shift - 1); + temp3R = (mid3 - side3) << (shift - 1); temp0L >>= 16; temp1L >>= 16; temp2L >>= 16; @@ -91236,26 +92190,26 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_mid_side__scalar(ma ma_uint32 temp1R; ma_uint32 temp2R; ma_uint32 temp3R; - ma_uint32 mid0 = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 mid1 = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 mid2 = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 mid3 = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - ma_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - ma_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - ma_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + ma_uint64 mid0 = (ma_uint64)(ma_int64)pInputSamples0[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint64 mid1 = (ma_uint64)(ma_int64)pInputSamples0[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint64 mid2 = (ma_uint64)(ma_int64)pInputSamples0[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint64 mid3 = (ma_uint64)(ma_int64)pInputSamples0[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint64 side0 = (ma_uint64)(ma_int64)pInputSamples1[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + ma_uint64 side1 = (ma_uint64)(ma_int64)pInputSamples1[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + ma_uint64 side2 = (ma_uint64)(ma_int64)pInputSamples1[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + ma_uint64 side3 = (ma_uint64)(ma_int64)pInputSamples1[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; mid0 = (mid0 << 1) | (side0 & 0x01); mid1 = (mid1 << 1) | (side1 & 0x01); mid2 = (mid2 << 1) | (side2 & 0x01); mid3 = (mid3 << 1) | (side3 & 0x01); - temp0L = ((ma_int32)(mid0 + side0) >> 1); - temp1L = ((ma_int32)(mid1 + side1) >> 1); - temp2L = ((ma_int32)(mid2 + side2) >> 1); - temp3L = ((ma_int32)(mid3 + side3) >> 1); - temp0R = ((ma_int32)(mid0 - side0) >> 1); - temp1R = ((ma_int32)(mid1 - side1) >> 1); - temp2R = ((ma_int32)(mid2 - side2) >> 1); - temp3R = ((ma_int32)(mid3 - side3) >> 1); + temp0L = (ma_uint32)((ma_int64)(mid0 + side0) >> 1); + temp1L = (ma_uint32)((ma_int64)(mid1 + side1) >> 1); + temp2L = (ma_uint32)((ma_int64)(mid2 + side2) >> 1); + temp3L = (ma_uint32)((ma_int64)(mid3 + side3) >> 1); + temp0R = (ma_uint32)((ma_int64)(mid0 - side0) >> 1); + temp1R = (ma_uint32)((ma_int64)(mid1 - side1) >> 1); + temp2R = (ma_uint32)((ma_int64)(mid2 - side2) >> 1); + temp3R = (ma_uint32)((ma_int64)(mid3 - side3) >> 1); temp0L >>= 16; temp1L >>= 16; temp2L >>= 16; @@ -91274,12 +92228,24 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_mid_side__scalar(ma pOutputSamples[i*8+7] = (ma_int16)temp3R; } } - for (i = (frameCount4 << 2); i < frameCount; ++i) { - ma_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - mid = (mid << 1) | (side & 0x01); - pOutputSamples[i*2+0] = (ma_int16)(((ma_uint32)((ma_int32)(mid + side) >> 1) << unusedBitsPerSample) >> 16); - pOutputSamples[i*2+1] = (ma_int16)(((ma_uint32)((ma_int32)(mid - side) >> 1) << unusedBitsPerSample) >> 16); + if (shift > 0) { + for (i = (frameCount4 << 2); i < frameCount; ++i) { + ma_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + mid = (mid << 1) | (side & 0x01); + pOutputSamples[i*2+0] = (ma_int16)(((ma_uint32)((ma_int32)(mid + side) >> 1) << unusedBitsPerSample) >> 16); + pOutputSamples[i*2+1] = (ma_int16)(((ma_uint32)((ma_int32)(mid - side) >> 1) << unusedBitsPerSample) >> 16); + } + } else { + for (i = (frameCount4 << 2); i < frameCount; ++i) { + ma_uint64 mid; + ma_uint64 side; + mid = (ma_uint64)(ma_int64)pInputSamples0[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + side = (ma_uint64)(ma_int64)pInputSamples1[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + mid = (mid << 1) | (side & 0x01); + pOutputSamples[i*2+0] = (ma_int16)((ma_int64)(mid + side) >> 17); + pOutputSamples[i*2+1] = (ma_int16)((ma_int64)(mid - side) >> 17); + } } } #if defined(MA_DR_FLAC_SUPPORT_SSE2) @@ -91290,7 +92256,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_mid_side__sse2(ma_d const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0; const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1; ma_uint32 shift = unusedBitsPerSample; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); if (shift == 0) { for (i = 0; i < frameCount4; ++i) { __m128i mid; @@ -91349,7 +92315,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_mid_side__neon(ma_d ma_uint32 shift = unusedBitsPerSample; int32x4_t wbpsShift0_4; int32x4_t wbpsShift1_4; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); wbpsShift0_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); wbpsShift1_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample); if (shift == 0) { @@ -91405,11 +92371,11 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_mid_side__neon(ma_d static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_mid_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples) { #if defined(MA_DR_FLAC_SUPPORT_SSE2) - if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsSSE2Supported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s16__decode_mid_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #elif defined(MA_DR_FLAC_SUPPORT_NEON) - if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsNEONSupported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s16__decode_mid_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #endif @@ -91520,11 +92486,11 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo_ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples) { #if defined(MA_DR_FLAC_SUPPORT_SSE2) - if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsSSE2Supported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #elif defined(MA_DR_FLAC_SUPPORT_NEON) - if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsNEONSupported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #endif @@ -91536,10 +92502,51 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo( #endif } } +static MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint64 iFirstPCMFrame, ma_int16* pOutputSamples) +{ + unsigned int channelCount; + ma_uint32 unusedBitsPerSample; + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample < 32); + channelCount = ma_dr_flac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment); + unusedBitsPerSample = 32 - pFlac->currentFLACFrame.header.bitsPerSample; + if (channelCount == 2) { + const ma_int32* pDecodedSamples0 = pFlac->currentFLACFrame.subframes[0].samples.pS32 + iFirstPCMFrame; + const ma_int32* pDecodedSamples1 = pFlac->currentFLACFrame.subframes[1].samples.pS32 + iFirstPCMFrame; + switch (pFlac->currentFLACFrame.header.channelAssignment) + { + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE: + { + ma_dr_flac_read_pcm_frames_s16__decode_left_side(pFlac, frameCount, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pOutputSamples); + } break; + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE: + { + ma_dr_flac_read_pcm_frames_s16__decode_right_side(pFlac, frameCount, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pOutputSamples); + } break; + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE: + { + ma_dr_flac_read_pcm_frames_s16__decode_mid_side(pFlac, frameCount, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pOutputSamples); + } break; + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_INDEPENDENT_STEREO: + { + ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo(pFlac, frameCount, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pOutputSamples); + } break; + default: MA_DR_FLAC_ASSERT(!"Invalid stereo channel assignment"); break; + } + } else { + ma_uint64 i; + for (i = 0; i < frameCount; ++i) { + unsigned int j; + for (j = 0; j < channelCount; ++j) { + ma_int32 sampleS32; + sampleS32 = (ma_int32)((ma_uint32)(pFlac->currentFLACFrame.subframes[j].samples.pS32[iFirstPCMFrame + i]) << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample)); + pOutputSamples[(i*channelCount)+j] = (ma_int16)(sampleS32 >> 16); + } + } + } +} MA_API ma_uint64 ma_dr_flac_read_pcm_frames_s16(ma_dr_flac* pFlac, ma_uint64 framesToRead, ma_int16* pBufferOut) { ma_uint64 framesRead; - ma_uint32 unusedBitsPerSample; if (pFlac == NULL || framesToRead == 0) { return 0; } @@ -91547,7 +92554,6 @@ MA_API ma_uint64 ma_dr_flac_read_pcm_frames_s16(ma_dr_flac* pFlac, ma_uint64 fra return ma_dr_flac__seek_forward_by_pcm_frames(pFlac, framesToRead); } MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 32); - unusedBitsPerSample = 32 - pFlac->bitsPerSample; framesRead = 0; while (framesToRead > 0) { if (pFlac->currentFLACFrame.pcmFramesRemaining == 0) { @@ -91561,43 +92567,15 @@ MA_API ma_uint64 ma_dr_flac_read_pcm_frames_s16(ma_dr_flac* pFlac, ma_uint64 fra if (frameCountThisIteration > pFlac->currentFLACFrame.pcmFramesRemaining) { frameCountThisIteration = pFlac->currentFLACFrame.pcmFramesRemaining; } - if (channelCount == 2) { - const ma_int32* pDecodedSamples0 = pFlac->currentFLACFrame.subframes[0].pSamplesS32 + iFirstPCMFrame; - const ma_int32* pDecodedSamples1 = pFlac->currentFLACFrame.subframes[1].pSamplesS32 + iFirstPCMFrame; - switch (pFlac->currentFLACFrame.header.channelAssignment) - { - case MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE: - { - ma_dr_flac_read_pcm_frames_s16__decode_left_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); - } break; - case MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE: - { - ma_dr_flac_read_pcm_frames_s16__decode_right_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); - } break; - case MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE: - { - ma_dr_flac_read_pcm_frames_s16__decode_mid_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); - } break; - case MA_DR_FLAC_CHANNEL_ASSIGNMENT_INDEPENDENT: - default: - { - ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); - } break; - } + if (pFlac->bitsPerSample != 32) { + ma_dr_flac_read_pcm_frames_s16__decode(pFlac, frameCountThisIteration, iFirstPCMFrame, pBufferOut); } else { - ma_uint64 i; - for (i = 0; i < frameCountThisIteration; ++i) { - unsigned int j; - for (j = 0; j < channelCount; ++j) { - ma_int32 sampleS32 = (ma_int32)((ma_uint32)(pFlac->currentFLACFrame.subframes[j].pSamplesS32[iFirstPCMFrame + i]) << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample)); - pBufferOut[(i*channelCount)+j] = (ma_int16)(sampleS32 >> 16); - } - } + ma_dr_flac_read_pcm_frames_s16__decode_s64(pFlac, frameCountThisIteration, iFirstPCMFrame, pBufferOut); } - framesRead += frameCountThisIteration; - pBufferOut += frameCountThisIteration * channelCount; - framesToRead -= frameCountThisIteration; - pFlac->currentPCMFrame += frameCountThisIteration; + framesRead += frameCountThisIteration; + pBufferOut += frameCountThisIteration * channelCount; + framesToRead -= frameCountThisIteration; + pFlac->currentPCMFrame += frameCountThisIteration; pFlac->currentFLACFrame.pcmFramesRemaining -= (ma_uint32)frameCountThisIteration; } } @@ -91665,7 +92643,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_left_side__sse2(ma_ ma_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8; ma_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8; __m128 factor; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); factor = _mm_set1_ps(1.0f / 8388608.0f); for (i = 0; i < frameCount4; ++i) { __m128i left = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); @@ -91697,7 +92675,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_left_side__neon(ma_ float32x4_t factor4; int32x4_t shift0_4; int32x4_t shift1_4; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); factor4 = vdupq_n_f32(1.0f / 8388608.0f); shift0_4 = vdupq_n_s32(shift0); shift1_4 = vdupq_n_s32(shift1); @@ -91723,14 +92701,32 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_left_side__neon(ma_ } } #endif +static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_duplicate_mono(ma_uint64 frameCount, ma_uint32 shift, const ma_int32* pInputSamples, float* pOutputSamples) +{ + ma_uint64 i; + const ma_uint32* pInputSamplesU32 = (const ma_uint32*)pInputSamples; + float factor; + MA_DR_FLAC_ASSERT(shift < 32); + factor = 1 / 2147483648.0f; + for (i = 0; i < frameCount; ++i) { + float sample; + sample = (ma_int32)(pInputSamplesU32[i] << shift) * factor; + pOutputSamples[i*2+0] = sample; + pOutputSamples[i*2+1] = sample; + } +} static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_left_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples) { + if (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample == 32) { + ma_dr_flac_read_pcm_frames_f32__decode_duplicate_mono(frameCount, unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample, pInputSamples0, pOutputSamples); + return; + } #if defined(MA_DR_FLAC_SUPPORT_SSE2) - if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsSSE2Supported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_f32__decode_left_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #elif defined(MA_DR_FLAC_SUPPORT_NEON) - if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsNEONSupported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_f32__decode_left_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #endif @@ -91804,7 +92800,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_right_side__sse2(ma ma_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8; ma_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8; __m128 factor; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); factor = _mm_set1_ps(1.0f / 8388608.0f); for (i = 0; i < frameCount4; ++i) { __m128i side = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0); @@ -91836,7 +92832,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_right_side__neon(ma float32x4_t factor4; int32x4_t shift0_4; int32x4_t shift1_4; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); factor4 = vdupq_n_f32(1.0f / 8388608.0f); shift0_4 = vdupq_n_s32(shift0); shift1_4 = vdupq_n_s32(shift1); @@ -91864,12 +92860,16 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_right_side__neon(ma #endif static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_right_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples) { + if (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample == 32) { + ma_dr_flac_read_pcm_frames_f32__decode_duplicate_mono(frameCount, unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample, pInputSamples1, pOutputSamples); + return; + } #if defined(MA_DR_FLAC_SUPPORT_SSE2) - if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsSSE2Supported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_f32__decode_right_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #elif defined(MA_DR_FLAC_SUPPORT_NEON) - if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsNEONSupported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_f32__decode_right_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #endif @@ -91902,7 +92902,6 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_mid_side__scalar(ma ma_uint32 shift = unusedBitsPerSample; float factor = 1 / 2147483648.0; if (shift > 0) { - shift -= 1; for (i = 0; i < frameCount4; ++i) { ma_uint32 temp0L; ma_uint32 temp1L; @@ -91924,14 +92923,14 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_mid_side__scalar(ma mid1 = (mid1 << 1) | (side1 & 0x01); mid2 = (mid2 << 1) | (side2 & 0x01); mid3 = (mid3 << 1) | (side3 & 0x01); - temp0L = (mid0 + side0) << shift; - temp1L = (mid1 + side1) << shift; - temp2L = (mid2 + side2) << shift; - temp3L = (mid3 + side3) << shift; - temp0R = (mid0 - side0) << shift; - temp1R = (mid1 - side1) << shift; - temp2R = (mid2 - side2) << shift; - temp3R = (mid3 - side3) << shift; + temp0L = (mid0 + side0) << (shift - 1); + temp1L = (mid1 + side1) << (shift - 1); + temp2L = (mid2 + side2) << (shift - 1); + temp3L = (mid3 + side3) << (shift - 1); + temp0R = (mid0 - side0) << (shift - 1); + temp1R = (mid1 - side1) << (shift - 1); + temp2R = (mid2 - side2) << (shift - 1); + temp3R = (mid3 - side3) << (shift - 1); pOutputSamples[i*8+0] = (ma_int32)temp0L * factor; pOutputSamples[i*8+1] = (ma_int32)temp0R * factor; pOutputSamples[i*8+2] = (ma_int32)temp1L * factor; @@ -91951,26 +92950,26 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_mid_side__scalar(ma ma_uint32 temp1R; ma_uint32 temp2R; ma_uint32 temp3R; - ma_uint32 mid0 = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 mid1 = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 mid2 = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 mid3 = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - ma_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - ma_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - ma_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + ma_uint64 mid0 = (ma_uint64)(ma_int64)pInputSamples0[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint64 mid1 = (ma_uint64)(ma_int64)pInputSamples0[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint64 mid2 = (ma_uint64)(ma_int64)pInputSamples0[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint64 mid3 = (ma_uint64)(ma_int64)pInputSamples0[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint64 side0 = (ma_uint64)(ma_int64)pInputSamples1[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + ma_uint64 side1 = (ma_uint64)(ma_int64)pInputSamples1[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + ma_uint64 side2 = (ma_uint64)(ma_int64)pInputSamples1[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + ma_uint64 side3 = (ma_uint64)(ma_int64)pInputSamples1[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; mid0 = (mid0 << 1) | (side0 & 0x01); mid1 = (mid1 << 1) | (side1 & 0x01); mid2 = (mid2 << 1) | (side2 & 0x01); mid3 = (mid3 << 1) | (side3 & 0x01); - temp0L = (ma_uint32)((ma_int32)(mid0 + side0) >> 1); - temp1L = (ma_uint32)((ma_int32)(mid1 + side1) >> 1); - temp2L = (ma_uint32)((ma_int32)(mid2 + side2) >> 1); - temp3L = (ma_uint32)((ma_int32)(mid3 + side3) >> 1); - temp0R = (ma_uint32)((ma_int32)(mid0 - side0) >> 1); - temp1R = (ma_uint32)((ma_int32)(mid1 - side1) >> 1); - temp2R = (ma_uint32)((ma_int32)(mid2 - side2) >> 1); - temp3R = (ma_uint32)((ma_int32)(mid3 - side3) >> 1); + temp0L = (ma_uint32)((ma_int64)(mid0 + side0) >> 1); + temp1L = (ma_uint32)((ma_int64)(mid1 + side1) >> 1); + temp2L = (ma_uint32)((ma_int64)(mid2 + side2) >> 1); + temp3L = (ma_uint32)((ma_int64)(mid3 + side3) >> 1); + temp0R = (ma_uint32)((ma_int64)(mid0 - side0) >> 1); + temp1R = (ma_uint32)((ma_int64)(mid1 - side1) >> 1); + temp2R = (ma_uint32)((ma_int64)(mid2 - side2) >> 1); + temp3R = (ma_uint32)((ma_int64)(mid3 - side3) >> 1); pOutputSamples[i*8+0] = (ma_int32)temp0L * factor; pOutputSamples[i*8+1] = (ma_int32)temp0R * factor; pOutputSamples[i*8+2] = (ma_int32)temp1L * factor; @@ -91981,12 +92980,24 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_mid_side__scalar(ma pOutputSamples[i*8+7] = (ma_int32)temp3R * factor; } } - for (i = (frameCount4 << 2); i < frameCount; ++i) { - ma_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; - ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; - mid = (mid << 1) | (side & 0x01); - pOutputSamples[i*2+0] = (ma_int32)((ma_uint32)((ma_int32)(mid + side) >> 1) << unusedBitsPerSample) * factor; - pOutputSamples[i*2+1] = (ma_int32)((ma_uint32)((ma_int32)(mid - side) >> 1) << unusedBitsPerSample) * factor; + if (shift > 0) { + for (i = (frameCount4 << 2); i < frameCount; ++i) { + ma_uint32 mid = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + mid = (mid << 1) | (side & 0x01); + pOutputSamples[i*2+0] = (ma_int32)((ma_uint32)((ma_int32)(mid + side) >> 1) << unusedBitsPerSample) * factor; + pOutputSamples[i*2+1] = (ma_int32)((ma_uint32)((ma_int32)(mid - side) >> 1) << unusedBitsPerSample) * factor; + } + } else { + for (i = (frameCount4 << 2); i < frameCount; ++i) { + ma_uint64 mid; + ma_uint64 side; + mid = (ma_uint64)(ma_int64)pInputSamples0[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample; + side = (ma_uint64)(ma_int64)pInputSamples1[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample; + mid = (mid << 1) | (side & 0x01); + pOutputSamples[i*2+0] = (ma_int32)((ma_int64)(mid + side) >> 1) * factor; + pOutputSamples[i*2+1] = (ma_int32)((ma_int64)(mid - side) >> 1) * factor; + } } } #if defined(MA_DR_FLAC_SUPPORT_SSE2) @@ -91999,7 +93010,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_mid_side__sse2(ma_d ma_uint32 shift = unusedBitsPerSample - 8; float factor; __m128 factor128; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); factor = 1.0f / 8388608.0f; factor128 = _mm_set1_ps(factor); if (shift == 0) { @@ -92069,7 +93080,7 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_mid_side__neon(ma_d int32x4_t shift4; int32x4_t wbps0_4; int32x4_t wbps1_4; - MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24); + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample <= 24); factor = 1.0f / 8388608.0f; factor4 = vdupq_n_f32(factor); wbps0_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample); @@ -92128,11 +93139,11 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_mid_side__neon(ma_d static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_mid_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples) { #if defined(MA_DR_FLAC_SUPPORT_SSE2) - if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsSSE2Supported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_f32__decode_mid_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #elif defined(MA_DR_FLAC_SUPPORT_NEON) - if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsNEONSupported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_f32__decode_mid_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #endif @@ -92247,11 +93258,11 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo_ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples) { #if defined(MA_DR_FLAC_SUPPORT_SSE2) - if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsSSE2Supported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #elif defined(MA_DR_FLAC_SUPPORT_NEON) - if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) { + if (ma_dr_flac__gIsNEONSupported && pFlac->currentFLACFrame.header.bitsPerSample <= 24) { ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples); } else #endif @@ -92263,10 +93274,51 @@ static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo( #endif } } +static MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint64 iFirstPCMFrame, float* pOutputSamples) +{ + unsigned int channelCount; + ma_uint32 unusedBitsPerSample; + MA_DR_FLAC_ASSERT(pFlac->currentFLACFrame.header.bitsPerSample < 32); + channelCount = ma_dr_flac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment); + unusedBitsPerSample = 32 - pFlac->currentFLACFrame.header.bitsPerSample; + if (channelCount == 2) { + const ma_int32* pDecodedSamples0 = pFlac->currentFLACFrame.subframes[0].samples.pS32 + iFirstPCMFrame; + const ma_int32* pDecodedSamples1 = pFlac->currentFLACFrame.subframes[1].samples.pS32 + iFirstPCMFrame; + switch (pFlac->currentFLACFrame.header.channelAssignment) + { + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE: + { + ma_dr_flac_read_pcm_frames_f32__decode_left_side(pFlac, frameCount, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pOutputSamples); + } break; + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE: + { + ma_dr_flac_read_pcm_frames_f32__decode_right_side(pFlac, frameCount, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pOutputSamples); + } break; + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE: + { + ma_dr_flac_read_pcm_frames_f32__decode_mid_side(pFlac, frameCount, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pOutputSamples); + } break; + case MA_DR_FLAC_CHANNEL_ASSIGNMENT_INDEPENDENT_STEREO: + { + ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo(pFlac, frameCount, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pOutputSamples); + } break; + default: MA_DR_FLAC_ASSERT(!"Invalid stereo channel assignment"); break; + } + } else { + ma_uint64 i; + for (i = 0; i < frameCount; ++i) { + unsigned int j; + for (j = 0; j < channelCount; ++j) { + ma_int32 sampleS32; + sampleS32 = (ma_int32)((ma_uint32)(pFlac->currentFLACFrame.subframes[j].samples.pS32[iFirstPCMFrame + i]) << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample)); + pOutputSamples[(i*channelCount)+j] = (float)(sampleS32 / 2147483648.0); + } + } + } +} MA_API ma_uint64 ma_dr_flac_read_pcm_frames_f32(ma_dr_flac* pFlac, ma_uint64 framesToRead, float* pBufferOut) { ma_uint64 framesRead; - ma_uint32 unusedBitsPerSample; if (pFlac == NULL || framesToRead == 0) { return 0; } @@ -92274,7 +93326,6 @@ MA_API ma_uint64 ma_dr_flac_read_pcm_frames_f32(ma_dr_flac* pFlac, ma_uint64 fra return ma_dr_flac__seek_forward_by_pcm_frames(pFlac, framesToRead); } MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 32); - unusedBitsPerSample = 32 - pFlac->bitsPerSample; framesRead = 0; while (framesToRead > 0) { if (pFlac->currentFLACFrame.pcmFramesRemaining == 0) { @@ -92288,43 +93339,15 @@ MA_API ma_uint64 ma_dr_flac_read_pcm_frames_f32(ma_dr_flac* pFlac, ma_uint64 fra if (frameCountThisIteration > pFlac->currentFLACFrame.pcmFramesRemaining) { frameCountThisIteration = pFlac->currentFLACFrame.pcmFramesRemaining; } - if (channelCount == 2) { - const ma_int32* pDecodedSamples0 = pFlac->currentFLACFrame.subframes[0].pSamplesS32 + iFirstPCMFrame; - const ma_int32* pDecodedSamples1 = pFlac->currentFLACFrame.subframes[1].pSamplesS32 + iFirstPCMFrame; - switch (pFlac->currentFLACFrame.header.channelAssignment) - { - case MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE: - { - ma_dr_flac_read_pcm_frames_f32__decode_left_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); - } break; - case MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE: - { - ma_dr_flac_read_pcm_frames_f32__decode_right_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); - } break; - case MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE: - { - ma_dr_flac_read_pcm_frames_f32__decode_mid_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); - } break; - case MA_DR_FLAC_CHANNEL_ASSIGNMENT_INDEPENDENT: - default: - { - ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut); - } break; - } + if (pFlac->bitsPerSample != 32) { + ma_dr_flac_read_pcm_frames_f32__decode(pFlac, frameCountThisIteration, iFirstPCMFrame, pBufferOut); } else { - ma_uint64 i; - for (i = 0; i < frameCountThisIteration; ++i) { - unsigned int j; - for (j = 0; j < channelCount; ++j) { - ma_int32 sampleS32 = (ma_int32)((ma_uint32)(pFlac->currentFLACFrame.subframes[j].pSamplesS32[iFirstPCMFrame + i]) << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample)); - pBufferOut[(i*channelCount)+j] = (float)(sampleS32 / 2147483648.0); - } - } + ma_dr_flac_read_pcm_frames_f32__decode_s64(pFlac, frameCountThisIteration, iFirstPCMFrame, pBufferOut); } - framesRead += frameCountThisIteration; - pBufferOut += frameCountThisIteration * channelCount; - framesToRead -= frameCountThisIteration; - pFlac->currentPCMFrame += frameCountThisIteration; + framesRead += frameCountThisIteration; + pBufferOut += frameCountThisIteration * channelCount; + framesToRead -= frameCountThisIteration; + pFlac->currentPCMFrame += frameCountThisIteration; pFlac->currentFLACFrame.pcmFramesRemaining -= (unsigned int)frameCountThisIteration; } } @@ -92338,33 +93361,32 @@ MA_API ma_bool32 ma_dr_flac_seek_to_pcm_frame(ma_dr_flac* pFlac, ma_uint64 pcmFr if (pFlac->currentPCMFrame == pcmFrameIndex) { return MA_TRUE; } - if (pFlac->firstFLACFramePosInBytes == 0) { - return MA_FALSE; - } if (pcmFrameIndex == 0) { pFlac->currentPCMFrame = 0; return ma_dr_flac__seek_to_first_frame(pFlac); } else { ma_bool32 wasSuccessful = MA_FALSE; ma_uint64 originalPCMFrame = pFlac->currentPCMFrame; - if (pcmFrameIndex > pFlac->totalPCMFrameCount) { + if (pFlac->totalPCMFrameCount != 0 && pcmFrameIndex > pFlac->totalPCMFrameCount) { pcmFrameIndex = pFlac->totalPCMFrameCount; } - if (pcmFrameIndex > pFlac->currentPCMFrame) { - ma_uint32 offset = (ma_uint32)(pcmFrameIndex - pFlac->currentPCMFrame); - if (pFlac->currentFLACFrame.pcmFramesRemaining > offset) { - pFlac->currentFLACFrame.pcmFramesRemaining -= offset; - pFlac->currentPCMFrame = pcmFrameIndex; - return MA_TRUE; - } - } else { - ma_uint32 offsetAbs = (ma_uint32)(pFlac->currentPCMFrame - pcmFrameIndex); - ma_uint32 currentFLACFramePCMFrameCount = pFlac->currentFLACFrame.header.blockSizeInPCMFrames; - ma_uint32 currentFLACFramePCMFramesConsumed = currentFLACFramePCMFrameCount - pFlac->currentFLACFrame.pcmFramesRemaining; - if (currentFLACFramePCMFramesConsumed > offsetAbs) { - pFlac->currentFLACFrame.pcmFramesRemaining += offsetAbs; - pFlac->currentPCMFrame = pcmFrameIndex; - return MA_TRUE; + if (ma_dr_flac__is_current_flac_frame_valid(pFlac)) { + if (pcmFrameIndex > pFlac->currentPCMFrame) { + ma_uint64 offset = pcmFrameIndex - pFlac->currentPCMFrame; + if (pFlac->currentFLACFrame.pcmFramesRemaining > offset) { + pFlac->currentFLACFrame.pcmFramesRemaining -= (ma_uint32)offset; + pFlac->currentPCMFrame = pcmFrameIndex; + return MA_TRUE; + } + } else { + ma_uint64 offsetAbs = pFlac->currentPCMFrame - pcmFrameIndex; + ma_uint32 currentFLACFramePCMFrameCount = pFlac->currentFLACFrame.header.blockSizeInPCMFrames; + ma_uint32 currentFLACFramePCMFramesConsumed = currentFLACFramePCMFrameCount - pFlac->currentFLACFrame.pcmFramesRemaining; + if (currentFLACFramePCMFramesConsumed > offsetAbs) { + pFlac->currentFLACFrame.pcmFramesRemaining += (ma_uint32)offsetAbs; + pFlac->currentPCMFrame = pcmFrameIndex; + return MA_TRUE; + } } } #ifndef MA_DR_FLAC_NO_OGG @@ -92390,8 +93412,9 @@ MA_API ma_bool32 ma_dr_flac_seek_to_pcm_frame(ma_dr_flac* pFlac, ma_uint64 pcmFr if (wasSuccessful) { pFlac->currentPCMFrame = pcmFrameIndex; } else { - if (ma_dr_flac_seek_to_pcm_frame(pFlac, originalPCMFrame) == MA_FALSE) { - ma_dr_flac_seek_to_pcm_frame(pFlac, 0); + ma_dr_flac__seek_to_first_frame(pFlac); + if (originalPCMFrame != 0 && ma_dr_flac_seek_to_pcm_frame(pFlac, originalPCMFrame) == MA_FALSE) { + ma_dr_flac__seek_to_first_frame(pFlac); } } return wasSuccessful; @@ -92677,7 +93700,7 @@ MA_API ma_bool32 ma_dr_flac_next_cuesheet_track(ma_dr_flac_cuesheet_track_iterat MA_DR_FLAC_COPY_MEMORY(cuesheetTrack.ISRC, pRunningData, sizeof(cuesheetTrack.ISRC)); pRunningData += 12; cuesheetTrack.isAudio = (pRunningData[0] & 0x80) != 0; cuesheetTrack.preEmphasis = (pRunningData[0] & 0x40) != 0; pRunningData += 14; - cuesheetTrack.indexCount = pRunningData[0]; pRunningData += 1; + cuesheetTrack.indexCount = pRunningData[0]; pRunningData += 1 + (MA_DR_FLAC_CUESHEET_TRACK_PADDED_SIZE_IN_BYTES - MA_DR_FLAC_CUESHEET_TRACK_SIZE_IN_BYTES); cuesheetTrack.pIndexPoints = (const ma_dr_flac_cuesheet_track_index*)pRunningData; pRunningData += cuesheetTrack.indexCount * sizeof(ma_dr_flac_cuesheet_track_index); pIter->pRunningData = pRunningData; pIter->countRemaining -= 1; @@ -92753,7 +93776,10 @@ MA_API const char* ma_dr_mp3_version_string(void) #define MA_DR_MP3_MIN(a, b) ((a) > (b) ? (b) : (a)) #define MA_DR_MP3_MAX(a, b) ((a) < (b) ? (b) : (a)) #if !defined(MA_DR_MP3_NO_SIMD) -#if !defined(MA_DR_MP3_ONLY_SIMD) && (defined(_M_X64) || defined(__x86_64__) || defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) +#if !defined(MA_DR_MP3_ONLY_SIMD) && ((defined(_MSC_VER) && _MSC_VER >= 1400) && defined(_M_X64)) || ((defined(__i386) || defined(_M_IX86) || defined(__i386__) || defined(__x86_64__)) && ((defined(_M_IX86_FP) && _M_IX86_FP == 2) || defined(__SSE2__))) +#define MA_DR_MP3_ONLY_SIMD +#endif +#if !defined(MA_DR_MP3_ONLY_SIMD) && (defined(__ARM_NEON) || defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) #define MA_DR_MP3_ONLY_SIMD #endif #if ((defined(_MSC_VER) && _MSC_VER >= 1400) && defined(_M_X64)) || ((defined(__i386) || defined(_M_IX86) || defined(__i386__) || defined(__x86_64__)) && ((defined(_M_IX86_FP) && _M_IX86_FP == 2) || defined(__SSE2__))) @@ -94270,6 +95296,7 @@ MA_API int ma_dr_mp3dec_decode_frame(ma_dr_mp3dec *dec, const ma_uint8 *mp3, int int i = 0, igr, frame_size = 0, success = 1; const ma_uint8 *hdr; ma_dr_mp3_bs bs_frame[1]; + info->layer = 0; if (mp3_bytes > 4 && dec->header[0] == 0xff && ma_dr_mp3_hdr_compare(dec->header, mp3)) { frame_size = ma_dr_mp3_hdr_frame_bytes(mp3, dec->free_format_bytes) + ma_dr_mp3_hdr_padding(mp3); @@ -94406,7 +95433,7 @@ MA_API void ma_dr_mp3dec_f32_to_s16(const float *in, ma_int16 *out, size_t num_s } } #ifndef MA_DR_MP3_SEEK_LEADING_MP3_FRAMES -#define MA_DR_MP3_SEEK_LEADING_MP3_FRAMES 2 +#define MA_DR_MP3_SEEK_LEADING_MP3_FRAMES 16 #endif #define MA_DR_MP3_MIN_DATA_CHUNK_SIZE 16384 #ifndef MA_DR_MP3_DATA_CHUNK_SIZE @@ -94671,6 +95698,8 @@ static ma_uint32 ma_dr_mp3_decode_next_frame_ex__callbacks(ma_dr_mp3* pMP3, ma_d return 0; } pMP3->dataSize += bytesRead; + } else if (info.layer != 0) { + pMP3->mp3FramesSkipped += 1; } }; return pcmFramesRead; @@ -94700,6 +95729,9 @@ static ma_uint32 ma_dr_mp3_decode_next_frame_ex__memory(ma_dr_mp3* pMP3, ma_dr_m } break; } else if (info.frame_bytes > 0) { + if (info.layer != 0) { + pMP3->mp3FramesSkipped += 1; + } pMP3->memory.currentReadPos += (size_t)info.frame_bytes; pMP3->streamCursor += (size_t)info.frame_bytes; } else { @@ -94712,6 +95744,7 @@ static ma_uint32 ma_dr_mp3_decode_next_frame_ex__memory(ma_dr_mp3* pMP3, ma_dr_m } static ma_uint32 ma_dr_mp3_decode_next_frame_ex(ma_dr_mp3* pMP3, ma_dr_mp3d_sample_t* pPCMFrames, ma_dr_mp3dec_frame_info* pMP3FrameInfo, const ma_uint8** ppMP3FrameData) { + pMP3->mp3FramesSkipped = 0; if (pMP3->memory.pData != NULL && pMP3->memory.dataSize > 0) { return ma_dr_mp3_decode_next_frame_ex__memory(pMP3, pPCMFrames, pMP3FrameInfo, ppMP3FrameData); } else { @@ -94884,6 +95917,14 @@ static ma_bool32 ma_dr_mp3_init_internal(ma_dr_mp3* pMP3, ma_dr_mp3_read_proc on return MA_FALSE; } } else { + pMP3->pData = (ma_uint8*)ma_dr_mp3__malloc_from_callbacks(MA_DR_MP3_DATA_CHUNK_SIZE, &pMP3->allocationCallbacks); + if (pMP3->pData == NULL) { + return MA_FALSE; + } + pMP3->dataCapacity = MA_DR_MP3_DATA_CHUNK_SIZE; + MA_DR_MP3_COPY_MEMORY(pMP3->pData, header, sizeof(header)); + pMP3->dataSize = sizeof(header); + pMP3->streamCursor = sizeof(header); } } } else { @@ -94908,8 +95949,19 @@ static ma_bool32 ma_dr_mp3_init_internal(ma_dr_mp3* pMP3, ma_dr_mp3_read_proc on ma_bool32 isInfo = MA_FALSE; const ma_uint8* pTagData; const ma_uint8* pTagDataBeg; + const void* pDataBufferEnd = NULL; + size_t frameBytes; pTagDataBeg = pFirstFrameData + MA_DR_MP3_HDR_SIZE + (bs.pos/8); pTagData = pTagDataBeg; + if (pMP3->memory.pData != NULL && pMP3->memory.dataSize > 0) { + pDataBufferEnd = pMP3->memory.pData + pMP3->memory.dataSize; + } else { + pDataBufferEnd = pMP3->pData + pMP3->dataCapacity; + } + frameBytes = MA_DR_MP3_MIN((size_t)firstFrameInfo.frame_bytes, (size_t)((ma_uint8*)pDataBufferEnd - pFirstFrameData)); + if (frameBytes < (size_t)(pTagData - pFirstFrameData) + 8) { + goto done_xing_info; + } isXing = (pTagData[0] == 'X' && pTagData[1] == 'i' && pTagData[2] == 'n' && pTagData[3] == 'g'); isInfo = (pTagData[0] == 'I' && pTagData[1] == 'n' && pTagData[2] == 'f' && pTagData[3] == 'o'); if (isXing || isInfo) { @@ -94917,33 +95969,49 @@ static ma_bool32 ma_dr_mp3_init_internal(ma_dr_mp3* pMP3, ma_dr_mp3_read_proc on ma_uint32 flags = pTagData[7]; pTagData += 8; if (flags & 0x01) { + if (frameBytes < (size_t)(pTagData - pFirstFrameData) + 4) { + goto done_xing_info; + } detectedMP3FrameCount = (ma_uint32)pTagData[0] << 24 | (ma_uint32)pTagData[1] << 16 | (ma_uint32)pTagData[2] << 8 | (ma_uint32)pTagData[3]; pTagData += 4; } if (flags & 0x02) { + if (frameBytes < (size_t)(pTagData - pFirstFrameData) + 4) { + goto done_xing_info; + } bytes = (ma_uint32)pTagData[0] << 24 | (ma_uint32)pTagData[1] << 16 | (ma_uint32)pTagData[2] << 8 | (ma_uint32)pTagData[3]; (void)bytes; pTagData += 4; } if (flags & 0x04) { + if (frameBytes < (size_t)(pTagData - pFirstFrameData) + 100) { + goto done_xing_info; + } pTagData += 100; } if (flags & 0x08) { + if (frameBytes < (size_t)(pTagData - pFirstFrameData) + 4) { + goto done_xing_info; + } pTagData += 4; } + if (frameBytes < (size_t)(pTagData - pFirstFrameData) + 1) { + goto done_xing_info; + } if (pTagData[0]) { + int delayInPCMFrames; + int paddingInPCMFrames; + if (frameBytes < (size_t)(pTagData - pFirstFrameData) + 36) { + goto done_xing_info; + } pTagData += 21; - if (pTagData - pFirstFrameData + 14 < firstFrameInfo.frame_bytes) { - int delayInPCMFrames; - int paddingInPCMFrames; - delayInPCMFrames = (( (ma_uint32)pTagData[0] << 4) | ((ma_uint32)pTagData[1] >> 4)) + (528 + 1); - paddingInPCMFrames = ((((ma_uint32)pTagData[1] & 0xF) << 8) | ((ma_uint32)pTagData[2] )) - (528 + 1); - if (paddingInPCMFrames < 0) { - paddingInPCMFrames = 0; - } - pMP3->delayInPCMFrames = (ma_uint32)delayInPCMFrames; - pMP3->paddingInPCMFrames = (ma_uint32)paddingInPCMFrames; + delayInPCMFrames = (( (ma_uint32)pTagData[0] << 4) | ((ma_uint32)pTagData[1] >> 4)) + (528 + 1); + paddingInPCMFrames = ((((ma_uint32)pTagData[1] & 0xF) << 8) | ((ma_uint32)pTagData[2] )) - (528 + 1); + if (paddingInPCMFrames < 0) { + paddingInPCMFrames = 0; } + pMP3->delayInPCMFrames = (ma_uint32)delayInPCMFrames; + pMP3->paddingInPCMFrames = (ma_uint32)paddingInPCMFrames; } if (isXing) { pMP3->isVBR = MA_TRUE; @@ -94959,9 +96027,9 @@ static ma_bool32 ma_dr_mp3_init_internal(ma_dr_mp3* pMP3, ma_dr_mp3_read_proc on } pMP3->pcmFramesRemainingInMP3Frame = 0; pMP3->streamStartOffset += (ma_uint32)(firstFrameInfo.frame_bytes); - pMP3->streamCursor = pMP3->streamStartOffset; ma_dr_mp3dec_init(&pMP3->decoder); } + done_xing_info:; } else { } } @@ -94971,7 +96039,7 @@ static ma_bool32 ma_dr_mp3_init_internal(ma_dr_mp3* pMP3, ma_dr_mp3_read_proc on return MA_FALSE; } if (detectedMP3FrameCount != 0xFFFFFFFF) { - pMP3->totalPCMFrameCount = detectedMP3FrameCount * firstFramePCMFrameCount; + pMP3->totalPCMFrameCount = (ma_uint64)detectedMP3FrameCount * firstFramePCMFrameCount; } pMP3->channels = pMP3->mp3FrameChannels; pMP3->sampleRate = pMP3->mp3FrameSampleRate; @@ -95271,7 +96339,7 @@ MA_API ma_uint64 ma_dr_mp3_read_pcm_frames_f32(ma_dr_mp3* pMP3, ma_uint64 frames return ma_dr_mp3_read_pcm_frames_raw(pMP3, framesToRead, pBufferOut); #else { - ma_int16 pTempS16[8192]; + ma_int16 pTempS16[1152*2]; ma_uint64 totalPCMFramesRead = 0; while (totalPCMFramesRead < framesToRead) { ma_uint64 framesJustRead; @@ -95300,7 +96368,7 @@ MA_API ma_uint64 ma_dr_mp3_read_pcm_frames_s16(ma_dr_mp3* pMP3, ma_uint64 frames return ma_dr_mp3_read_pcm_frames_raw(pMP3, framesToRead, pBufferOut); #else { - float pTempF32[4096]; + float pTempF32[1152*2]; ma_uint64 totalPCMFramesRead = 0; while (totalPCMFramesRead < framesToRead) { ma_uint64 framesJustRead; @@ -95340,6 +96408,13 @@ static ma_bool32 ma_dr_mp3_seek_to_start_of_stream(ma_dr_mp3* pMP3) ma_dr_mp3_reset(pMP3); return MA_TRUE; } +static ma_uint64 ma_dr_mp3_get_cursor_in_pcm_frames(const ma_dr_mp3* pMP3) +{ + if (pMP3->currentPCMFrame > pMP3->delayInPCMFrames) { + return pMP3->currentPCMFrame - pMP3->delayInPCMFrames; + } + return 0; +} static ma_bool32 ma_dr_mp3_seek_forward_by_pcm_frames__brute_force(ma_dr_mp3* pMP3, ma_uint64 frameOffset) { ma_uint64 framesRead; @@ -95355,17 +96430,20 @@ static ma_bool32 ma_dr_mp3_seek_forward_by_pcm_frames__brute_force(ma_dr_mp3* pM } static ma_bool32 ma_dr_mp3_seek_to_pcm_frame__brute_force(ma_dr_mp3* pMP3, ma_uint64 frameIndex) { + ma_uint64 cursor; MA_DR_MP3_ASSERT(pMP3 != NULL); - if (frameIndex == pMP3->currentPCMFrame) { + cursor = ma_dr_mp3_get_cursor_in_pcm_frames(pMP3); + if (frameIndex == cursor) { return MA_TRUE; } - if (frameIndex < pMP3->currentPCMFrame) { + if (frameIndex < cursor) { if (!ma_dr_mp3_seek_to_start_of_stream(pMP3)) { return MA_FALSE; } + cursor = 0; } - MA_DR_MP3_ASSERT(frameIndex >= pMP3->currentPCMFrame); - return ma_dr_mp3_seek_forward_by_pcm_frames__brute_force(pMP3, (frameIndex - pMP3->currentPCMFrame)); + MA_DR_MP3_ASSERT(frameIndex >= cursor); + return ma_dr_mp3_seek_forward_by_pcm_frames__brute_force(pMP3, (frameIndex - cursor)); } static ma_bool32 ma_dr_mp3_find_closest_seek_point(ma_dr_mp3* pMP3, ma_uint64 frameIndex, ma_uint32* pSeekPointIndex) { @@ -95387,15 +96465,22 @@ static ma_bool32 ma_dr_mp3_seek_to_pcm_frame__seek_table(ma_dr_mp3* pMP3, ma_uin { ma_dr_mp3_seek_point seekPoint; ma_uint32 priorSeekPointIndex; - ma_uint16 iMP3Frame; + ma_uint32 iMP3Frame; + ma_uint32 mp3FramesDecoded; + ma_uint64 internalFrameIndex; ma_uint64 leftoverFrames; MA_DR_MP3_ASSERT(pMP3 != NULL); MA_DR_MP3_ASSERT(pMP3->pSeekPoints != NULL); MA_DR_MP3_ASSERT(pMP3->seekPointCount > 0); - if (ma_dr_mp3_find_closest_seek_point(pMP3, frameIndex, &priorSeekPointIndex)) { + if (frameIndex > MA_UINT64_MAX - pMP3->delayInPCMFrames) { + internalFrameIndex = MA_UINT64_MAX; + } else { + internalFrameIndex = frameIndex + pMP3->delayInPCMFrames; + } + if (ma_dr_mp3_find_closest_seek_point(pMP3, internalFrameIndex, &priorSeekPointIndex)) { seekPoint = pMP3->pSeekPoints[priorSeekPointIndex]; } else { - seekPoint.seekPosInBytes = 0; + seekPoint.seekPosInBytes = pMP3->streamStartOffset; seekPoint.pcmFrameIndex = 0; seekPoint.mp3FramesToDiscard = 0; seekPoint.pcmFramesToDiscard = 0; @@ -95404,20 +96489,29 @@ static ma_bool32 ma_dr_mp3_seek_to_pcm_frame__seek_table(ma_dr_mp3* pMP3, ma_uin return MA_FALSE; } ma_dr_mp3_reset(pMP3); - for (iMP3Frame = 0; iMP3Frame < seekPoint.mp3FramesToDiscard; ++iMP3Frame) { - ma_uint32 pcmFramesRead; - ma_dr_mp3d_sample_t* pPCMFrames; - pPCMFrames = NULL; - if (iMP3Frame == seekPoint.mp3FramesToDiscard-1) { - pPCMFrames = (ma_dr_mp3d_sample_t*)pMP3->pcmFrames; - } - pcmFramesRead = ma_dr_mp3_decode_next_frame_ex(pMP3, pPCMFrames, NULL, NULL); - if (pcmFramesRead == 0) { + iMP3Frame = 0; + mp3FramesDecoded = 0; + while (iMP3Frame < seekPoint.mp3FramesToDiscard) { + if (ma_dr_mp3_decode_next_frame_ex(pMP3, (ma_dr_mp3d_sample_t*)pMP3->pcmFrames, NULL, NULL) == 0) { + break; + } + iMP3Frame += 1 + pMP3->mp3FramesSkipped; + if (pMP3->mp3FramesSkipped == 0) { + mp3FramesDecoded += 1; + } else { + mp3FramesDecoded = 1; + } + } + pMP3->pcmFramesConsumedInMP3Frame = 0; + pMP3->pcmFramesRemainingInMP3Frame = 0; + if (iMP3Frame != seekPoint.mp3FramesToDiscard || (mp3FramesDecoded < 2 && seekPoint.seekPosInBytes != pMP3->streamStartOffset)) { + if (!ma_dr_mp3_seek_to_start_of_stream(pMP3)) { return MA_FALSE; } + return ma_dr_mp3_seek_forward_by_pcm_frames__brute_force(pMP3, frameIndex); } pMP3->currentPCMFrame = seekPoint.pcmFrameIndex - seekPoint.pcmFramesToDiscard; - leftoverFrames = frameIndex - pMP3->currentPCMFrame; + leftoverFrames = frameIndex - ma_dr_mp3_get_cursor_in_pcm_frames(pMP3); return ma_dr_mp3_seek_forward_by_pcm_frames__brute_force(pMP3, leftoverFrames); } MA_API ma_bool32 ma_dr_mp3_seek_to_pcm_frame(ma_dr_mp3* pMP3, ma_uint64 frameIndex) @@ -95445,7 +96539,7 @@ MA_API ma_bool32 ma_dr_mp3_get_mp3_and_pcm_frame_count(ma_dr_mp3* pMP3, ma_uint6 if (pMP3->onSeek == NULL) { return MA_FALSE; } - currentPCMFrame = pMP3->currentPCMFrame; + currentPCMFrame = ma_dr_mp3_get_cursor_in_pcm_frames(pMP3); if (!ma_dr_mp3_seek_to_start_of_stream(pMP3)) { return MA_FALSE; } @@ -95522,6 +96616,7 @@ typedef struct { ma_uint64 bytePos; ma_uint64 pcmFrameIndex; + ma_uint64 mp3FrameIndex; } ma_dr_mp3__seeking_mp3_frame_info; MA_API ma_bool32 ma_dr_mp3_calculate_seek_points(ma_dr_mp3* pMP3, ma_uint32* pSeekPointCount, ma_dr_mp3_seek_point* pSeekPoints) { @@ -95536,23 +96631,24 @@ MA_API ma_bool32 ma_dr_mp3_calculate_seek_points(ma_dr_mp3* pMP3, ma_uint32* pSe if (seekPointCount == 0) { return MA_FALSE; } - currentPCMFrame = pMP3->currentPCMFrame; + currentPCMFrame = ma_dr_mp3_get_cursor_in_pcm_frames(pMP3); if (!ma_dr_mp3_get_mp3_and_pcm_frame_count(pMP3, &totalMP3FrameCount, &totalPCMFrameCount)) { return MA_FALSE; } if (totalMP3FrameCount < MA_DR_MP3_SEEK_LEADING_MP3_FRAMES+1) { seekPointCount = 1; - pSeekPoints[0].seekPosInBytes = 0; + pSeekPoints[0].seekPosInBytes = pMP3->streamStartOffset; pSeekPoints[0].pcmFrameIndex = 0; pSeekPoints[0].mp3FramesToDiscard = 0; pSeekPoints[0].pcmFramesToDiscard = 0; } else { ma_uint64 pcmFramesBetweenSeekPoints; ma_dr_mp3__seeking_mp3_frame_info mp3FrameInfo[MA_DR_MP3_SEEK_LEADING_MP3_FRAMES+1]; + ma_uint32 mp3FrameInfoCount = 0; ma_uint64 runningPCMFrameCount = 0; float runningPCMFrameCountFractionalPart = 0; + ma_uint64 runningMP3FrameCount = 0; ma_uint64 nextTargetPCMFrame; - ma_uint32 iMP3Frame; ma_uint32 iSeekPoint; if (seekPointCount > totalMP3FrameCount-1) { seekPointCount = (ma_uint32)totalMP3FrameCount-1; @@ -95561,46 +96657,35 @@ MA_API ma_bool32 ma_dr_mp3_calculate_seek_points(ma_dr_mp3* pMP3, ma_uint32* pSe if (!ma_dr_mp3_seek_to_start_of_stream(pMP3)) { return MA_FALSE; } - for (iMP3Frame = 0; iMP3Frame < MA_DR_MP3_SEEK_LEADING_MP3_FRAMES+1; ++iMP3Frame) { - ma_uint32 pcmFramesInCurrentMP3FrameIn; - MA_DR_MP3_ASSERT(pMP3->streamCursor >= pMP3->dataSize); - mp3FrameInfo[iMP3Frame].bytePos = pMP3->streamCursor - pMP3->dataSize; - mp3FrameInfo[iMP3Frame].pcmFrameIndex = runningPCMFrameCount; - pcmFramesInCurrentMP3FrameIn = ma_dr_mp3_decode_next_frame_ex(pMP3, NULL, NULL, NULL); - if (pcmFramesInCurrentMP3FrameIn == 0) { - return MA_FALSE; - } - ma_dr_mp3__accumulate_running_pcm_frame_count(pMP3, pcmFramesInCurrentMP3FrameIn, &runningPCMFrameCount, &runningPCMFrameCountFractionalPart); - } nextTargetPCMFrame = 0; for (iSeekPoint = 0; iSeekPoint < seekPointCount; ++iSeekPoint) { nextTargetPCMFrame += pcmFramesBetweenSeekPoints; - for (;;) { - if (nextTargetPCMFrame < runningPCMFrameCount) { - pSeekPoints[iSeekPoint].seekPosInBytes = mp3FrameInfo[0].bytePos; - pSeekPoints[iSeekPoint].pcmFrameIndex = nextTargetPCMFrame; - pSeekPoints[iSeekPoint].mp3FramesToDiscard = MA_DR_MP3_SEEK_LEADING_MP3_FRAMES; - pSeekPoints[iSeekPoint].pcmFramesToDiscard = (ma_uint16)(nextTargetPCMFrame - mp3FrameInfo[MA_DR_MP3_SEEK_LEADING_MP3_FRAMES-1].pcmFrameIndex); - break; - } else { + while (mp3FrameInfoCount == 0 || nextTargetPCMFrame >= runningPCMFrameCount) { + ma_uint32 pcmFramesInCurrentMP3FrameIn; + if (mp3FrameInfoCount == MA_DR_MP3_COUNTOF(mp3FrameInfo)) { size_t i; - ma_uint32 pcmFramesInCurrentMP3FrameIn; for (i = 0; i < MA_DR_MP3_COUNTOF(mp3FrameInfo)-1; ++i) { mp3FrameInfo[i] = mp3FrameInfo[i+1]; } - mp3FrameInfo[MA_DR_MP3_COUNTOF(mp3FrameInfo)-1].bytePos = pMP3->streamCursor - pMP3->dataSize; - mp3FrameInfo[MA_DR_MP3_COUNTOF(mp3FrameInfo)-1].pcmFrameIndex = runningPCMFrameCount; - pcmFramesInCurrentMP3FrameIn = ma_dr_mp3_decode_next_frame_ex(pMP3, NULL, NULL, NULL); - if (pcmFramesInCurrentMP3FrameIn == 0) { - pSeekPoints[iSeekPoint].seekPosInBytes = mp3FrameInfo[0].bytePos; - pSeekPoints[iSeekPoint].pcmFrameIndex = nextTargetPCMFrame; - pSeekPoints[iSeekPoint].mp3FramesToDiscard = MA_DR_MP3_SEEK_LEADING_MP3_FRAMES; - pSeekPoints[iSeekPoint].pcmFramesToDiscard = (ma_uint16)(nextTargetPCMFrame - mp3FrameInfo[MA_DR_MP3_SEEK_LEADING_MP3_FRAMES-1].pcmFrameIndex); - break; - } - ma_dr_mp3__accumulate_running_pcm_frame_count(pMP3, pcmFramesInCurrentMP3FrameIn, &runningPCMFrameCount, &runningPCMFrameCountFractionalPart); + mp3FrameInfoCount -= 1; + } + MA_DR_MP3_ASSERT(pMP3->streamCursor >= pMP3->dataSize); + mp3FrameInfo[mp3FrameInfoCount].bytePos = pMP3->streamCursor - pMP3->dataSize; + mp3FrameInfo[mp3FrameInfoCount].pcmFrameIndex = runningPCMFrameCount; + mp3FrameInfo[mp3FrameInfoCount].mp3FrameIndex = runningMP3FrameCount; + pcmFramesInCurrentMP3FrameIn = ma_dr_mp3_decode_next_frame_ex(pMP3, NULL, NULL, NULL); + if (pcmFramesInCurrentMP3FrameIn == 0) { + break; } + mp3FrameInfoCount += 1; + runningMP3FrameCount += 1 + pMP3->mp3FramesSkipped; + ma_dr_mp3__accumulate_running_pcm_frame_count(pMP3, pcmFramesInCurrentMP3FrameIn, &runningPCMFrameCount, &runningPCMFrameCountFractionalPart); } + MA_DR_MP3_ASSERT(mp3FrameInfoCount > 0); + pSeekPoints[iSeekPoint].seekPosInBytes = mp3FrameInfo[0].bytePos; + pSeekPoints[iSeekPoint].pcmFrameIndex = nextTargetPCMFrame; + pSeekPoints[iSeekPoint].mp3FramesToDiscard = (ma_uint16)(mp3FrameInfo[mp3FrameInfoCount-1].mp3FrameIndex - mp3FrameInfo[0].mp3FrameIndex); + pSeekPoints[iSeekPoint].pcmFramesToDiscard = (ma_uint16)(nextTargetPCMFrame - mp3FrameInfo[mp3FrameInfoCount-1].pcmFrameIndex); } if (!ma_dr_mp3_seek_to_start_of_stream(pMP3)) { return MA_FALSE; @@ -95631,7 +96716,7 @@ static float* ma_dr_mp3__full_read_and_close_f32(ma_dr_mp3* pMP3, ma_dr_mp3_conf ma_uint64 totalFramesRead = 0; ma_uint64 framesCapacity = 0; float* pFrames = NULL; - float temp[4096]; + float temp[1152*2]; MA_DR_MP3_ASSERT(pMP3 != NULL); for (;;) { ma_uint64 framesToReadRightNow = MA_DR_MP3_COUNTOF(temp) / pMP3->channels; @@ -95684,7 +96769,7 @@ static ma_int16* ma_dr_mp3__full_read_and_close_s16(ma_dr_mp3* pMP3, ma_dr_mp3_c ma_uint64 totalFramesRead = 0; ma_uint64 framesCapacity = 0; ma_int16* pFrames = NULL; - ma_int16 temp[4096]; + ma_int16 temp[1152*2]; MA_DR_MP3_ASSERT(pMP3 != NULL); for (;;) { ma_uint64 framesToReadRightNow = MA_DR_MP3_COUNTOF(temp) / pMP3->channels; From bc428e407a52bd3e21b5985cabeef57cdff9bc4a Mon Sep 17 00:00:00 2001 From: Tarek Dakhran Date: Sat, 31 Oct 2026 23:40:50 +0100 Subject: [PATCH 07/36] llama : enable tensor split for LFM3 (#31431) Assisted-by: Claude Opus 5.5 --- src/llama-arch.cpp | 1 - src/llama-model.cpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 8a0d6b124f4e..5b2318409923 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -1375,7 +1375,6 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: case LLM_ARCH_GRANITE_HYBRID: - case LLM_ARCH_LFM3: case LLM_ARCH_MINIMAX_01: case LLM_ARCH_MINIMAX_M2: case LLM_ARCH_MINIMAX_M3: diff --git a/src/llama-model.cpp b/src/llama-model.cpp index c4a5b64fd231..f45d30c38b18 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -591,7 +591,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight"); } if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) { - if (ud->model->arch == LLM_ARCH_LFM2 || ud->model->arch == LLM_ARCH_LFM2MOE) { + if (ud->model->arch == LLM_ARCH_LFM2 || ud->model->arch == LLM_ARCH_LFM2MOE || ud->model->arch == LLM_ARCH_LFM3) { // the LFM2 shortconv block runs fully mirrored, so its conv state must be mirrored too return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED, ""); } From 79bce0c8812ddfe4716922d061b67edc2ed517e9 Mon Sep 17 00:00:00 2001 From: Niklas Wenzel Date: Sat, 31 Oct 2026 23:55:02 +0100 Subject: [PATCH 08/36] tests : enable Gemma 4 in `test-llama-archs` (#31401) * tests : enable Gemma 4 in `test-llama-archs` * tests : use GQA for Gemma 4 in `test-llama-archs` --- src/llama-model-saver.cpp | 9 ++++++++- tests/fusion/MTL.csv | 2 ++ tests/test-llama-archs.cpp | 31 +++++++++++++------------------ 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index a08e173e9eed..36f3850d5180 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -199,6 +199,9 @@ void llama_model_saver::add_kv_from_model() { if (hparams.n_embd_out_impl > 0) { add_kv(LLM_KV_EMBEDDING_LENGTH_OUT, hparams.n_embd_out_impl); } + if (hparams.n_embd_per_layer > 0) { + add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer); + } add_kv(LLM_KV_BLOCK_COUNT, hparams.n_layer_all); add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); add_kv(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, true); @@ -307,7 +310,9 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL, hparams.indexer_kpool_select_tail); add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks); add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, true); - if (hparams.n_layer_kv_from_start < 0) { + if (hparams.n_layer_kv_from_start >= 0) { + add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(hparams.n_layer_all - hparams.n_layer_kv_from_start)); + } else { for (uint32_t il = 0; il < hparams.n_layer(); ++il) { if (hparams.kv_source(il) != il) { add_kv(LLM_KV_ATTENTION_KV_SOURCE_MAP, hparams.kv_source_impl, true); @@ -520,6 +525,8 @@ void llama_model_saver::add_tensors_from_model() { add_tensor(model->hc_head_base); add_tensor(model->hc_head_scale); add_tensor(model->per_layer_tok_embd); + add_tensor(model->per_layer_model_proj); + add_tensor(model->per_layer_proj_norm); add_tensor(model->value_embd); add_tensor(model->hc_head_norm); add_tensor(model->hc_head_down); diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv index e08d0c81b1f1..569947a7fbc8 100644 --- a/tests/fusion/MTL.csv +++ b/tests/fusion/MTL.csv @@ -81,6 +81,8 @@ gemma2 ,0 ,any ,RMS_NORM+MUL , 5 gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 gemma3 ,0 ,any ,RMS_NORM+MUL , 9 gemma3 ,0 ,any ,RMS_NORM+MUL+ADD , 4 +gemma4 ,0 ,any ,RMS_NORM+MUL , 47 +gemma4 ,0 ,any ,RMS_NORM+MUL+ADD , 20 glm-dsa ,0 ,any ,ADD+ADD , 1 glm-dsa ,0 ,any ,MUL+ADD , 2 glm-dsa ,0 ,any ,NORM+MUL+ADD , 2 diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index d82d6db79f8e..312733ecd22e 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -243,10 +243,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { n_head = 4; n_layer = 4; // dense and MoE layers, each with local and full attention } else if (arch == LLM_ARCH_GEMMA4) { - n_embd = 128; - n_head = 2; - n_ff = 192; - n_layer = 5; // need at least 5 for swa_pattern (every 5th is full_attention) + // TODO: expand coverage to both GQA and MQA (see https://github.com/ggml-org/llama.cpp/pull/31401#pullrequestreview-5710165130) + n_head = 4; + n_layer = 10; // last 5 layers reuse KV, both SWA and full attention } else if (arch == LLM_ARCH_GEMMA3N) { n_embd = 64; n_head = 1; @@ -305,7 +304,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { if (arch == LLM_ARCH_QWEN3) { n_head_kv = 1; // MQA coverage } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER || - arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) { + arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE || arch == LLM_ARCH_GEMMA4) { n_head_kv = 2; // GQA coverage } if (arch == LLM_ARCH_BEAM) { @@ -433,12 +432,17 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern); } else if (arch == LLM_ARCH_GEMMA4) { ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, n_embd/2); - ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(0)); + ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(5)); ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, n_embd_head); ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, n_embd_head); ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f); // SWA pattern: every 5th layer is full attention (matches E2B layer_types) - ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5)); + std::vector pattern; + pattern.reserve(n_layer); + for (uint32_t il = 0; il < n_layer; il++) { + pattern.push_back((il + 1) % 5 != 0); + } + ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern); } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT) { @@ -1001,9 +1005,6 @@ static bool arch_supported(const llm_arch arch) { if (arch == LLM_ARCH_WAVTOKENIZER_DEC) { return false; // FIXME CUDA backend crashes. } - if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) { - return false; // FIXME @ngxson - } if (arch == LLM_ARCH_GRANITE_SWITCH) { return false; // FIXME adapter fixture } @@ -1075,10 +1076,7 @@ static int save_models(const std::string & arch_filter, const size_t seed, const if (!arch_matches(arch_filter, arch)) { continue; } - if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) { - continue; // FIXME: ISWA KV cache initialization needs more fixture params - } - if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) { + if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH || arch == LLM_ARCH_GEMMA4_ASSISTANT) { continue; } for (bool moe : {false, true}) { @@ -1235,10 +1233,7 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con if (!arch_matches(arch_filter, arch)) { continue; } - if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) { - continue; // FIXME: ISWA KV cache initialization needs more fixture params - } - if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) { + if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH || arch == LLM_ARCH_GEMMA4_ASSISTANT) { continue; } From d5ef408e20c02ff015c2580de25a1860d4d1fbd7 Mon Sep 17 00:00:00 2001 From: bosh <98094229+boshjerns@users.noreply.github.com> Date: Sun, 1 Nov 2026 06:30:01 +0700 Subject: [PATCH 09/36] mtmd : fix InternVL CLS token order (#30886) --- tools/mtmd/models/internvl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/mtmd/models/internvl.cpp b/tools/mtmd/models/internvl.cpp index 65d7d5a6b7ba..af8f12cdc93d 100644 --- a/tools/mtmd/models/internvl.cpp +++ b/tools/mtmd/models/internvl.cpp @@ -10,7 +10,7 @@ ggml_cgraph * clip_graph_internvl::build() { // add CLS token ggml_tensor * cls_repeated = ggml_repeat_4d(ctx0, model.class_embedding, model.class_embedding->ne[0], 1, n_batch, 1); - inp = ggml_concat(ctx0, inp, cls_repeated, 1); + inp = ggml_concat(ctx0, cls_repeated, inp, 1); // The larger models use a different ViT, which uses RMS norm instead of layer norm // ref: https://github.com/ggml-org/llama.cpp/pull/13443#issuecomment-2869786188 @@ -28,7 +28,7 @@ ggml_cgraph * clip_graph_internvl::build() { // remove CLS token cur = ggml_view_3d(ctx0, cur, n_embd, n_patches, n_batch, - cur->nb[1], cur->nb[2], 0); + cur->nb[1], cur->nb[2], cur->nb[1]); cur = ggml_cont(ctx0, cur); // pixel shuffle From a91eab67d8389d979d29fdf6633b2edd0718eb4b Mon Sep 17 00:00:00 2001 From: Mark Todorovich Date: Sat, 31 Oct 2026 16:49:23 -0700 Subject: [PATCH 10/36] hexagon : fix DMA transfers with strides > 24 bits (#31442) * hexagon : fix DMA transfers with strides > 24 bits On v75+ the 2D DMA descriptor stores row_size and strides in 24 bits. dma_queue_push sent oversized strides to fallback_2d, which only splits nrows, so strides of 16 MB or more were truncated - wrong addresses. Fix will fall back to one transfer per row in that case. Also added two CONT tests with source strides over 16 MB. * hex-dma: refactor the fallback paths to remove duplication --------- Co-authored-by: Max Krasnyansky --- ggml/src/ggml-hexagon/htp/dma-queue.c | 69 ++++++++------------------- ggml/src/ggml-hexagon/htp/dma-queue.h | 11 +---- tests/test-backend-ops.cpp | 4 ++ 3 files changed, 27 insertions(+), 57 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.c b/ggml/src/ggml-hexagon/htp/dma-queue.c index ef61d2d242e6..11d52a919ac5 100644 --- a/ggml/src/ggml-hexagon/htp/dma-queue.c +++ b/ggml/src/ggml-hexagon/htp/dma-queue.c @@ -141,43 +141,9 @@ bool dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride dma_ring_flush(r1); r0->tail = r1->tail; - return dma_ring_push_single_2d(r0, ddata, 0, 0, 0, /*nrows=*/ 0); -} - -bool dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total) { - dma_ring * r0 = q->ring0; - dma_ring * r1 = q->ring1; - - if (((r0->push_idx + 1) & r0->idx_mask) == r0->pop_idx) { - return false; - } - - r1->tail = r0->tail; - - size_t rem_bytes = total; - dma_addr_t cur_dst = ddata.dst; - dma_addr_t cur_src = ddata.src; - - while (rem_bytes > 0) { - const uint32_t cur_bytes = MIN(rem_bytes, DMA_SAFE_CHUNK_SIZE); - dma_data cur_data = dma_make_data(cur_dst, cur_src); - if (!dma_ring_push_single_contig(r1, cur_data, cur_bytes)) { - dma_ring_flush(r1); - dma_ring_push_single_contig(r1, cur_data, cur_bytes); - } - cur_dst += cur_bytes; - cur_src += cur_bytes; - rem_bytes -= cur_bytes; - } - - dma_ring_flush(r1); - r0->tail = r1->tail; - return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0); } -#if __HVX_ARCH__ < 75 - bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { dma_ring * r0 = q->ring0; dma_ring * r1 = q->ring1; @@ -188,19 +154,28 @@ bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride r1->tail = r0->tail; - size_t rem_rows = nrows; - dma_addr_t cur_dst = ddata.dst; - dma_addr_t cur_src = ddata.src; - - while (rem_rows > 0) { - dma_data cur_data = dma_make_data(cur_dst, cur_src); - if (!dma_ring_push_single_1d(r1, cur_data, row_size)) { - dma_ring_flush(r1); - dma_ring_push_single_1d(r1, cur_data, row_size); + dma_addr_t row_dst = ddata.dst; + dma_addr_t row_src = ddata.src; + + for (size_t i = 0; i < nrows; i++) { + size_t rem_bytes = row_size; + dma_addr_t cur_dst = row_dst; + dma_addr_t cur_src = row_src; + + while (rem_bytes > 0) { + const uint32_t cur_bytes = MIN(rem_bytes, DMA_SAFE_CHUNK_SIZE); + dma_data cur_data = dma_make_data(cur_dst, cur_src); + if (!dma_ring_push_single_contig(r1, cur_data, cur_bytes)) { + dma_ring_flush(r1); + dma_ring_push_single_contig(r1, cur_data, cur_bytes); + } + cur_dst += cur_bytes; + cur_src += cur_bytes; + rem_bytes -= cur_bytes; } - cur_dst += dst_stride; - cur_src += src_stride; - rem_rows -= 1; + + row_dst += dst_stride; + row_src += src_stride; } dma_ring_flush(r1); @@ -208,5 +183,3 @@ bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0); } - -#endif diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h index 05f37305d991..678fc61d7e45 100644 --- a/ggml/src/ggml-hexagon/htp/dma-queue.h +++ b/ggml/src/ggml-hexagon/htp/dma-queue.h @@ -148,10 +148,7 @@ dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q); void dma_queue_alias_free(dma_queue_t q); bool dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows); -bool dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total); -#if __HVX_ARCH__ < 75 bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows); -#endif // TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead // but those do not seem to always compiler properly. @@ -400,7 +397,7 @@ static inline bool dma_queue_push(dma_queue * q, dma_data ddata, size_t dst_stri if (__builtin_expect(total <= DMA_MAX_SIZE_24B, 1)) { return dma_ring_push_single_contig(q->ring0, ddata, total); } - return dma_queue_push_fallback_contig(q, ddata, total); + return dma_queue_push_fallback_1d(q, ddata, 0, 0, total, 1); } // 2. Hot path: Standard strided 2D (10-20% of calls) @@ -410,14 +407,10 @@ static inline bool dma_queue_push(dma_queue * q, dma_data ddata, size_t dst_stri } // 3. Cold path: Descriptor chunking fallbacks (< 0.1%) -#if __HVX_ARCH__ < 75 - if (row_size <= DMA_MAX_SIZE_16B && (src_stride | dst_stride) <= DMA_MAX_STRIDE_16B) { + if ((row_size | src_stride | dst_stride) <= DMA_MAX_2D_ROW_SIZE) { return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows); } return dma_queue_push_fallback_1d(q, ddata, dst_stride, src_stride, row_size, nrows); -#else - return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows); -#endif } static inline void dma_sync_read(dma_queue * dma_q, void * dst, dma_addr_t src, size_t bytes) { diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index f75ff11b9999..e4ebd7c5f2d7 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10831,6 +10831,10 @@ static std::vector> make_test_cases_eval() { } } + // source strides over 16 MB (24-bit DMA descriptor limit) + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {512, 8193, 2, 1}, false, {2, 1, 0, 3})); + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {256, 16385, 2, 1}, false, {0, 2, 1, 3})); + auto add_test_bin_bcast = [&](ggml_type type, std::array ne, std::array nr, bool perm1 = false, bool src_overlap = false) { for (auto op : {ggml_add, ggml_sub, ggml_mul, ggml_div}) { test_cases.emplace_back(new test_bin_bcast(op, type, ne, nr, 1, perm1, src_overlap)); From 060b2fea4ed528810ddc91d48fb22b3e4d63eb85 Mon Sep 17 00:00:00 2001 From: shaofeiqi Date: Sat, 31 Oct 2026 22:26:47 -0700 Subject: [PATCH 11/36] opencl: add initial iq3 support (#31338) * opencl: add iq3_s AOS mv Assisted-by: Codex * opencl: add iq3_s SOA mv and tiled mm Assisted-by: Codex * opencl: add iq3_s noshuffle gemm/gemv for adreno Co-authored-by: Shaofei Qi Assisted-by: Codex * opencl: add iq3_s dp4a for adreno Co-authored-by: Shaofei Qi Assisted-by: Codex * opencl: optimize iq3_s adreno decode and narrow batches Co-authored-by: Shaofei Qi Assisted-by: Codex * opencl: add iq3_xxs AOS mv Assisted-by: Codex * opencl: add iq3_xxs SOA mv and tiled mm Assisted-by: Codex * opencl: add iq3_xxs noshuffle gemm/gemv for adreno Co-authored-by: Shaofei Qi Assisted-by: Codex * opencl: add iq3_xxs dp4a for adreno Co-authored-by: Shaofei Qi Assisted-by: Codex * opencl: optimize iq3_xxs adreno decode and narrow batches Co-authored-by: Shaofei Qi Assisted-by: Codex --------- Co-authored-by: Li He --- ggml/src/ggml-opencl/CMakeLists.txt | 14 + ggml/src/ggml-opencl/ggml-opencl.cpp | 1458 ++++++++++++++++- ggml/src/ggml-opencl/kernels/cvt.cl | 105 ++ .../kernels/gemm_noshuffle_iq3_s_f32.cl | 92 ++ .../kernels/gemm_noshuffle_iq3_s_q8_1_dp4a.cl | 154 ++ .../kernels/gemm_noshuffle_iq3_xxs_f32.cl | 89 + .../gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl | 137 ++ .../kernels/gemv_noshuffle_iq3_s_f32.cl | 128 ++ .../kernels/gemv_noshuffle_iq3_xxs_f32.cl | 120 ++ ggml/src/ggml-opencl/kernels/iq3_s_grid.cl | 66 + ggml/src/ggml-opencl/kernels/iq3_xxs_grid.cl | 38 + .../kernels/mul_mm_iq3_s_f32_l4_lm.cl | 193 +++ .../kernels/mul_mm_iq3_xxs_f32_l4_lm.cl | 187 +++ .../ggml-opencl/kernels/mul_mv_iq3_s_f32.cl | 146 ++ .../kernels/mul_mv_iq3_s_f32_flat.cl | 147 ++ .../ggml-opencl/kernels/mul_mv_iq3_xxs_f32.cl | 145 ++ .../kernels/mul_mv_iq3_xxs_f32_flat.cl | 142 ++ 17 files changed, 3276 insertions(+), 85 deletions(-) create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_q8_1_dp4a.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_s_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_xxs_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/iq3_s_grid.cl create mode 100644 ggml/src/ggml-opencl/kernels/iq3_xxs_grid.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_iq3_s_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_iq3_xxs_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32_flat.cl diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 046e42acdf30..f905370a3ae9 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -124,6 +124,12 @@ set(GGML_OPENCL_KERNELS gemm_noshuffle_iq1_s_f32 gemm_noshuffle_iq1_s_q8_1_dp4a gemv_noshuffle_iq1_s_f32 + gemm_noshuffle_iq3_s_f32 + gemm_noshuffle_iq3_s_q8_1_dp4a + gemv_noshuffle_iq3_s_f32 + gemm_noshuffle_iq3_xxs_f32 + gemm_noshuffle_iq3_xxs_q8_1_dp4a + gemv_noshuffle_iq3_xxs_f32 gemm_noshuffle_q3_k_q8_1_dp4a gemv_noshuffle_q3_k_f32 mul_mv_q2_k_f32 @@ -150,6 +156,14 @@ set(GGML_OPENCL_KERNELS mul_mv_iq1_s_f32 mul_mv_iq1_s_f32_flat mul_mm_iq1_s_f32_l4_lm + iq3_xxs_grid + mul_mv_iq3_xxs_f32 + mul_mv_iq3_xxs_f32_flat + mul_mm_iq3_xxs_f32_l4_lm + iq3_s_grid + mul_mv_iq3_s_f32 + mul_mv_iq3_s_f32_flat + mul_mm_iq3_s_f32_l4_lm mul_mv_q3_k_f32_flat mul_mm_q3_k_f32_l4_lm mul_mv_q2_k_f32_flat diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 5b05c9b366a9..6cfd0c0f9538 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1051,6 +1051,28 @@ struct ggml_backend_opencl_context { ggml_cl_buffer iq1_s_grid_buffer; cl_mem iq1_s_grid_image = nullptr; cl_kernel kernel_mul_mv_iq1_s_f32; + cl_kernel kernel_convert_block_iq3_xxs, kernel_restore_block_iq3_xxs; + cl_kernel kernel_mul_mv_iq3_xxs_f32_flat; + cl_kernel kernel_mul_mm_iq3_xxs_f32_l4_lm; + cl_kernel kernel_gemv_noshuffle_iq3_xxs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq3_xxs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a_narrow = nullptr; + int iq3_xxs_mv_nsg = 4; + ggml_cl_buffer iq3_xxs_grid_buffer; + cl_mem iq3_xxs_grid_image = nullptr; + cl_kernel kernel_mul_mv_iq3_xxs_f32; + cl_kernel kernel_convert_block_iq3_s, kernel_restore_block_iq3_s; + cl_kernel kernel_mul_mv_iq3_s_f32_flat; + cl_kernel kernel_mul_mm_iq3_s_f32_l4_lm; + cl_kernel kernel_gemv_noshuffle_iq3_s_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq3_s_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq3_s_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_iq3_s_q8_1_dp4a_narrow = nullptr; + int iq3_s_mv_nsg = 4; + ggml_cl_buffer iq3_s_grid_buffer; + cl_mem iq3_s_grid_image = nullptr; + cl_kernel kernel_mul_mv_iq3_s_f32; cl_kernel kernel_mul_mv_q4_K_f32; cl_kernel kernel_mul_mv_q4_K_f32_flat; cl_kernel kernel_mul_mv_q5_K_f32; @@ -1429,6 +1451,12 @@ struct ggml_backend_opencl_context { if (iq1_s_grid_image) { CL_CHECK(clReleaseMemObject(iq1_s_grid_image)); } + if (iq3_xxs_grid_image) { + CL_CHECK(clReleaseMemObject(iq3_xxs_grid_image)); + } + if (iq3_s_grid_image) { + CL_CHECK(clReleaseMemObject(iq3_s_grid_image)); + } if (iq2_xxs_grid_image) { CL_CHECK(clReleaseMemObject(iq2_xxs_grid_image)); } @@ -2130,6 +2158,10 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_restore_block_iq1_s = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq1_s", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_iq1_m = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq1_m", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_iq1_m = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq1_m", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq3_xxs = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq3_xxs", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq3_xxs = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq3_xxs", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq3_s = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq3_s", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq3_s = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq3_s", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_K", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_K", &err), err)); @@ -2894,6 +2926,112 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string iq3_s_grid_src { + #include "iq3_s_grid.cl.h" + }; +#else + const std::string iq3_s_grid_src = read_file("iq3_s_grid.cl"); +#endif + + // mul_mv_iq3_s_f32_flat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_iq3_s_f32_flat.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_iq3_s_f32_flat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq3_s_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq3_s_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mm_iq3_s_f32_l4_lm + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_iq3_s_f32_l4_lm.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_iq3_s_f32_l4_lm.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_iq3_s_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq3_s_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_iq3_s_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_iq3_s_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_iq3_s_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq3_s_f32 = clCreateKernel(prog, "kernel_mul_mv_iq3_s_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string iq3_xxs_grid_src { + #include "iq3_xxs_grid.cl.h" + }; +#else + const std::string iq3_xxs_grid_src = read_file("iq3_xxs_grid.cl"); +#endif + + // mul_mv_iq3_xxs_f32_flat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_iq3_xxs_f32_flat.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_iq3_xxs_f32_flat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq3_xxs_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq3_xxs_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mm_iq3_xxs_f32_l4_lm + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_iq3_xxs_f32_l4_lm.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_iq3_xxs_f32_l4_lm.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_iq3_xxs_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq3_xxs_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_iq3_xxs_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_iq3_xxs_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_iq3_xxs_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq3_xxs_f32 = clCreateKernel(prog, "kernel_mul_mv_iq3_xxs_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mv_q3_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -4959,6 +5097,180 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_noshuffle_iq3_s_q8_1_dp4a + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_iq3_s_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_iq3_s_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq3_s_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq3_s_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq3_s_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq3_s_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_iq3_s_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_iq3_s_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_iq3_s_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq3_s_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq3_s_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_iq3_s_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_iq3_s_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_iq3_s_f32.cl"); +#endif + if (const char * env = getenv("GGML_OPENCL_IQ3_S_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->iq3_s_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DIQ3_S_MV_NSG=" + std::to_string(backend_ctx->iq3_s_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, (iq3_s_grid_src + kernel_src).c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq3_s_f32", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->iq3_s_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_iq3_s_f32 = kernel; + cl_kernel export_grid = clCreateKernel(prog, "kernel_iq3_s_grid_export", &err); + CL_CHECK(err); + backend_ctx->iq3_s_grid_buffer.allocate(backend_ctx->context, 512*sizeof(cl_uint)); + CL_CHECK(clSetKernelArg(export_grid, 0, sizeof(cl_mem), &backend_ctx->iq3_s_grid_buffer.buffer)); + size_t global[] = { 512 }; + size_t local[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, export_grid, 1, nullptr, global, local, 0, nullptr, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseKernel(export_grid)); + CL_CHECK(clReleaseProgram(prog)); + + cl_image_format format = { CL_R, CL_UNSIGNED_INT32 }; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = 512; + desc.buffer = backend_ctx->iq3_s_grid_buffer.buffer; + backend_ctx->iq3_s_grid_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &format, &desc, nullptr, &err); + CL_CHECK(err); + break; + } + CL_CHECK(clReleaseKernel(kernel)); + CL_CHECK(clReleaseProgram(prog)); + GGML_ASSERT(backend_ctx->iq3_s_mv_nsg > 1); + backend_ctx->iq3_s_mv_nsg /= 2; + } + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_iq3_xxs_q8_1_dp4a + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_iq3_xxs_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_iq3_xxs_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_iq3_xxs_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq3_xxs_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq3_xxs_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_iq3_xxs_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_iq3_xxs_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_iq3_xxs_f32.cl"); +#endif + if (const char * env = getenv("GGML_OPENCL_IQ3_XXS_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->iq3_xxs_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DIQ3_XXS_MV_NSG=" + std::to_string(backend_ctx->iq3_xxs_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, (iq3_xxs_grid_src + kernel_src).c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq3_xxs_f32", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->iq3_xxs_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_iq3_xxs_f32 = kernel; + cl_kernel export_grid = clCreateKernel(prog, "kernel_iq3_xxs_grid_export", &err); + CL_CHECK(err); + backend_ctx->iq3_xxs_grid_buffer.allocate(backend_ctx->context, 256*sizeof(cl_uint)); + CL_CHECK(clSetKernelArg(export_grid, 0, sizeof(cl_mem), &backend_ctx->iq3_xxs_grid_buffer.buffer)); + size_t global[] = { 256 }; + size_t local[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, export_grid, 1, nullptr, global, local, 0, nullptr, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseKernel(export_grid)); + CL_CHECK(clReleaseProgram(prog)); + + cl_image_format format = { CL_R, CL_UNSIGNED_INT32 }; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = 256; + desc.buffer = backend_ctx->iq3_xxs_grid_buffer.buffer; + backend_ctx->iq3_xxs_grid_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &format, &desc, nullptr, &err); + CL_CHECK(err); + break; + } + CL_CHECK(clReleaseKernel(kernel)); + CL_CHECK(clReleaseProgram(prog)); + GGML_ASSERT(backend_ctx->iq3_xxs_mv_nsg > 1); + backend_ctx->iq3_xxs_mv_nsg /= 2; + } + GGML_LOG_CONT("."); + } + // gemm_noshuffle_iq2_s_q8_1_dp4a { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -8800,10 +9112,11 @@ static void transpose_2d_as_32b( ggml_backend_opencl_context * backend_ctx, cl_mem src, cl_mem dst, size_t size, cl_int stride, cl_int rows, - bool blocking = true + bool blocking = true, + bool auto_local = false ) { transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, - src, dst, size, stride, rows, blocking); + src, dst, size, stride, rows, blocking, auto_local); } #endif // GGML_OPENCL_USE_ADRENO_KERNELS @@ -9354,6 +9667,49 @@ struct ggml_tensor_extra_cl_q3_K { } }; +struct ggml_tensor_extra_cl_iq3_xxs { + bool noshuffle = false; + cl_mem qs = nullptr; // grid indices + cl_mem sas = nullptr; // packed sign codes and scales + cl_mem d = nullptr; // super-block scales + + size_t size_qs = 0, size_sas = 0, size_d = 0; + + ~ggml_tensor_extra_cl_iq3_xxs() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (sas != nullptr) { CL_CHECK(clReleaseMemObject(sas)); sas = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_sas = size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_iq3_s { + bool noshuffle = false; + cl_mem qs = nullptr; // grid index low bits + cl_mem sg = nullptr; // sign bits + cl_mem qh = nullptr; // grid index high bits + cl_mem sc = nullptr; // packed scales + cl_mem d = nullptr; // super-block scales + + size_t size_qs = 0, size_sg = 0, size_qh = 0, size_sc = 0, size_d = 0; + + ~ggml_tensor_extra_cl_iq3_s() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (sg != nullptr) { CL_CHECK(clReleaseMemObject(sg)); sg = nullptr; } + if (qh != nullptr) { CL_CHECK(clReleaseMemObject(qh)); qh = nullptr; } + if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_sg = size_qh = size_sc = size_d = 0; + } +}; + + struct ggml_tensor_extra_cl_iq2_xxs { bool noshuffle = false; cl_mem qs = nullptr; // grid indices @@ -10588,14 +10944,14 @@ static bool ggml_cl_q3_k_use_noshuffle(const ggml_backend_opencl_context * backe #endif } -static bool ggml_cl_iq1_m_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +static bool ggml_cl_iq3_s_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { #if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) - if (const char * env = getenv("GGML_OPENCL_IQ1_M_NOSHUFFLE")) { + if (const char * env = getenv("GGML_OPENCL_IQ3_S_NOSHUFFLE")) { if (env[0] == '0') { return false; } } - return tensor->type == GGML_TYPE_IQ1_M && + return tensor->type == GGML_TYPE_IQ3_S && backend_ctx->gpu_family == ADRENO && ggml_is_contiguous(tensor) && !tensor->view_src && use_adreno_kernels(backend_ctx, tensor); @@ -10606,14 +10962,14 @@ static bool ggml_cl_iq1_m_use_noshuffle(const ggml_backend_opencl_context * back #endif } -static bool ggml_cl_iq1_s_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +static bool ggml_cl_iq3_xxs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { #if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) - if (const char * env = getenv("GGML_OPENCL_IQ1_S_NOSHUFFLE")) { + if (const char * env = getenv("GGML_OPENCL_IQ3_XXS_NOSHUFFLE")) { if (env[0] == '0') { return false; } } - return tensor->type == GGML_TYPE_IQ1_S && + return tensor->type == GGML_TYPE_IQ3_XXS && backend_ctx->gpu_family == ADRENO && ggml_is_contiguous(tensor) && !tensor->view_src && use_adreno_kernels(backend_ctx, tensor); @@ -10624,14 +10980,14 @@ static bool ggml_cl_iq1_s_use_noshuffle(const ggml_backend_opencl_context * back #endif } -static bool ggml_cl_q2_0_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +static bool ggml_cl_iq1_m_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { #if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) - if (const char * env = getenv("GGML_OPENCL_Q2_0_NOSHUFFLE")) { + if (const char * env = getenv("GGML_OPENCL_IQ1_M_NOSHUFFLE")) { if (env[0] == '0') { return false; } } - return tensor->type == GGML_TYPE_Q2_0 && + return tensor->type == GGML_TYPE_IQ1_M && backend_ctx->gpu_family == ADRENO && ggml_is_contiguous(tensor) && !tensor->view_src && use_adreno_kernels(backend_ctx, tensor); @@ -10642,14 +10998,14 @@ static bool ggml_cl_q2_0_use_noshuffle(const ggml_backend_opencl_context * backe #endif } -static bool ggml_cl_iq2_s_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +static bool ggml_cl_iq1_s_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { #if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) - if (const char * env = getenv("GGML_OPENCL_IQ2_S_NOSHUFFLE")) { + if (const char * env = getenv("GGML_OPENCL_IQ1_S_NOSHUFFLE")) { if (env[0] == '0') { return false; } } - return tensor->type == GGML_TYPE_IQ2_S && + return tensor->type == GGML_TYPE_IQ1_S && backend_ctx->gpu_family == ADRENO && ggml_is_contiguous(tensor) && !tensor->view_src && use_adreno_kernels(backend_ctx, tensor); @@ -10660,14 +11016,14 @@ static bool ggml_cl_iq2_s_use_noshuffle(const ggml_backend_opencl_context * back #endif } -static bool ggml_cl_iq2_xs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +static bool ggml_cl_q2_0_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { #if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) - if (const char * env = getenv("GGML_OPENCL_IQ2_XS_NOSHUFFLE")) { + if (const char * env = getenv("GGML_OPENCL_Q2_0_NOSHUFFLE")) { if (env[0] == '0') { return false; } } - return tensor->type == GGML_TYPE_IQ2_XS && + return tensor->type == GGML_TYPE_Q2_0 && backend_ctx->gpu_family == ADRENO && ggml_is_contiguous(tensor) && !tensor->view_src && use_adreno_kernels(backend_ctx, tensor); @@ -10678,9 +11034,45 @@ static bool ggml_cl_iq2_xs_use_noshuffle(const ggml_backend_opencl_context * bac #endif } -static bool ggml_cl_iq2_xxs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +static bool ggml_cl_iq2_s_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { #if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) - if (const char * env = getenv("GGML_OPENCL_IQ2_XXS_NOSHUFFLE")) { + if (const char * env = getenv("GGML_OPENCL_IQ2_S_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_IQ2_S && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_cl_iq2_xs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_IQ2_XS_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_IQ2_XS && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_cl_iq2_xxs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_IQ2_XXS_NOSHUFFLE")) { if (env[0] == '0') { return false; } @@ -11273,6 +11665,18 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te return false; } return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_IQ3_S) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_iq3_s_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_IQ3_XXS) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_iq3_xxs_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); } else if (op->src[0]->type == GGML_TYPE_Q3_K) { const ggml_tensor * weight = op->src[0]->view_src; if (weight && ggml_cl_q3_k_use_noshuffle(backend_ctx, weight)) { @@ -11686,6 +12090,12 @@ struct ggml_backend_opencl_buffer_context { for (ggml_tensor_extra_cl_iq1_s * e : temp_tensor_extras_iq1_s) { delete e; } + for (ggml_tensor_extra_cl_iq3_s * e : temp_tensor_extras_iq3_s) { + delete e; + } + for (ggml_tensor_extra_cl_iq3_xxs * e : temp_tensor_extras_iq3_xxs) { + delete e; + } for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { delete e; } @@ -11713,6 +12123,12 @@ struct ggml_backend_opencl_buffer_context { for (ggml_tensor_extra_cl_iq1_s * e : temp_tensor_extras_iq1_s_in_use) { delete e; } + for (ggml_tensor_extra_cl_iq3_s * e : temp_tensor_extras_iq3_s_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_iq3_xxs * e : temp_tensor_extras_iq3_xxs_in_use) { + delete e; + } for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) { delete e; } @@ -12006,6 +12422,36 @@ struct ggml_backend_opencl_buffer_context { return extra; } + ggml_tensor_extra_cl_iq3_s * ggml_opencl_alloc_temp_tensor_extra_iq3_s() { + ggml_tensor_extra_cl_iq3_s * extra; + if (temp_tensor_extras_iq3_s.empty()) { + extra = new ggml_tensor_extra_cl_iq3_s(); + } else { + extra = temp_tensor_extras_iq3_s.back(); + temp_tensor_extras_iq3_s.pop_back(); + } + + temp_tensor_extras_iq3_s_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_iq3_xxs * ggml_opencl_alloc_temp_tensor_extra_iq3_xxs() { + ggml_tensor_extra_cl_iq3_xxs * extra; + if (temp_tensor_extras_iq3_xxs.empty()) { + extra = new ggml_tensor_extra_cl_iq3_xxs(); + } else { + extra = temp_tensor_extras_iq3_xxs.back(); + temp_tensor_extras_iq3_xxs.pop_back(); + } + + temp_tensor_extras_iq3_xxs_in_use.push_back(extra); + + extra->reset(); + return extra; + } + ggml_tensor_extra_cl_q6_K * ggml_opencl_alloc_temp_tensor_extra_q6_K() { ggml_tensor_extra_cl_q6_K * extra; if (temp_tensor_extras_q6_K.empty()) { @@ -12122,6 +12568,16 @@ struct ggml_backend_opencl_buffer_context { } temp_tensor_extras_iq1_s_in_use.clear(); + for (ggml_tensor_extra_cl_iq3_s * e : temp_tensor_extras_iq3_s_in_use) { + temp_tensor_extras_iq3_s.push_back(e); + } + temp_tensor_extras_iq3_s_in_use.clear(); + + for (ggml_tensor_extra_cl_iq3_xxs * e : temp_tensor_extras_iq3_xxs_in_use) { + temp_tensor_extras_iq3_xxs.push_back(e); + } + temp_tensor_extras_iq3_xxs_in_use.clear(); + q8_0_soa_tensors.clear(); q4_0_soa_tensors.clear(); } @@ -12158,6 +12614,7 @@ struct ggml_backend_opencl_buffer_context { std::vector temp_tensor_extras_iq2_s; std::vector temp_tensor_extras_iq2_xs; std::vector temp_tensor_extras_iq2_xxs; + std::vector temp_tensor_extras_iq3_xxs; std::vector temp_tensor_extras_q2_K_in_use; std::vector temp_tensor_extras_q3_K_in_use; std::vector temp_tensor_extras_iq2_s_in_use; @@ -12169,6 +12626,9 @@ struct ggml_backend_opencl_buffer_context { std::vector temp_tensor_extras_iq1_m; std::vector temp_tensor_extras_iq1_m_in_use; std::vector temp_tensor_extras_iq1_s_in_use; + std::vector temp_tensor_extras_iq3_s; + std::vector temp_tensor_extras_iq3_s_in_use; + std::vector temp_tensor_extras_iq3_xxs_in_use; std::vector temp_tensor_extras_q6_K; std::vector temp_tensor_extras_q6_K_in_use; @@ -12413,7 +12873,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, const int k = tensor->ne[0]; transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); transpose_2d_as_8b(backend_ctx, extra->sc, extra->sc, size_sc, k/16, m, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->dm, extra->dm, size_dm, k/(int)blck, m, true, true); + transpose_2d_as_32b(backend_ctx, extra->dm, extra->dm, size_dm, k/(int)blck, m, true, true); } #endif @@ -12496,8 +12956,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, const int k = tensor->ne[0]; transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); transpose_2d_as_8b(backend_ctx, extra->hm, extra->hm, size_hm, k/8, m, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sc, extra->sc, size_sc, 3*k/(int)blck, m, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, extra->d, size_d, k/(int)blck, m, true, true); + transpose_2d_as_32b(backend_ctx, extra->sc, extra->sc, size_sc, 3*k/(int)blck, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/(int)blck, m, true, true); } #endif @@ -12510,6 +12970,168 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, return; } + if (tensor->type == GGML_TYPE_IQ3_S) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); + + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_iq3_s * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq3_s(); + extra->noshuffle = ggml_cl_iq3_s_use_noshuffle(backend_ctx, tensor); + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; + + const size_t size_qs = n_blk * 64; + const size_t size_sg = n_blk * 32; + const size_t size_qh = n_blk * 8; + const size_t size_sc = n_blk * 4; + const size_t size_d = n_blk * 2; + GGML_ASSERT(size_qs + size_sg + size_qh + size_sc + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_qs, backend_ctx->alignment); + region.size = size_sg; + CL_CHECK((extra->sg = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_sg, backend_ctx->alignment); + region.size = size_qh; + CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_qh, backend_ctx->alignment); + region.size = size_sc; + CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_sc, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_kernel kernel = backend_ctx->kernel_convert_block_iq3_s; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sg)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->sc)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb_arg)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->sg, extra->sg, size_sg, k/8, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, k/32, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->sc, extra->sc, size_sc, k/64, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/256, m, true, true); + } +#endif + + extra->size_qs = size_qs; + extra->size_sg = size_sg; + extra->size_qh = size_qh; + extra->size_sc = size_sc; + extra->size_d = size_d; + + tensor->extra = extra; + return; + } + + if (tensor->type == GGML_TYPE_IQ3_XXS) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); + + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_iq3_xxs * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq3_xxs(); + extra->noshuffle = ggml_cl_iq3_xxs_use_noshuffle(backend_ctx, tensor); + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; + + const size_t size_qs = n_blk * 64; + const size_t size_sas = n_blk * 32; + const size_t size_d = n_blk * 2; + GGML_ASSERT(size_qs + size_sas + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_qs, backend_ctx->alignment); + region.size = size_sas; + CL_CHECK((extra->sas = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_sas, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_kernel kernel = backend_ctx->kernel_convert_block_iq3_xxs; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sas)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); + transpose_2d_as_32b(backend_ctx, extra->sas, extra->sas, size_sas, k/32, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/256, m, true, true); + } +#endif + + extra->size_qs = size_qs; + extra->size_sas = size_sas; + extra->size_d = size_d; + + tensor->extra = extra; + return; + } + if (tensor->type == GGML_TYPE_IQ1_M) { GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; @@ -12571,7 +13193,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, const int k = tensor->ne[0]; transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/8, m, true, true); transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, k/16, m, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->sc, extra->sc, size_sc, k/64, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->sc, extra->sc, size_sc, k/64, m, true, true); } #endif @@ -12643,8 +13265,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, const int m = tensor->ne[1]; const int k = tensor->ne[0]; transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/8, m, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->qh, extra->qh, size_qh, k/32, m, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, extra->d, size_d, k/(int)blck, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->qh, extra->qh, size_qh, k/32, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/(int)blck, m, true, true); } #endif @@ -12731,7 +13353,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, transpose_2d_as_8b(backend_ctx, extra->sg, extra->sg, size_sg, k/8, m, true, true); transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, k/32, m, true, true); transpose_2d_as_8b(backend_ctx, extra->sc, extra->sc, size_sc, k/32, m, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, extra->d, size_d, k/(int)blck, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/(int)blck, m, true, true); } #endif @@ -12804,9 +13426,9 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, if (extra->noshuffle) { const int m = tensor->ne[1]; const int k = tensor->ne[0]; - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->qs, extra->qs, size_qs, k/8, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, k/8, m, true, true); transpose_2d_as_8b(backend_ctx, extra->sc, extra->sc, size_sc, k/32, m, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, extra->d, size_d, k/(int)blck, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/(int)blck, m, true, true); } #endif @@ -12878,8 +13500,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, const int m = tensor->ne[1]; const int k = tensor->ne[0]; transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/8, m, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sas, extra->sas, size_sas, k/32, m, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, extra->d, size_d, k/(int)blck, m, true, true); + transpose_2d_as_32b(backend_ctx, extra->sas, extra->sas, size_sas, k/32, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/(int)blck, m, true, true); } #endif @@ -14671,7 +15293,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const int k = tensor->ne[0]; transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); transpose_2d_as_8b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/16, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->dm, dm, extra->size_dm, m, k/(int)blck, true, true); + transpose_2d_as_32b(backend_ctx, extra->dm, dm, extra->size_dm, m, k/(int)blck, true, true); } #endif cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_q2_k_noshuffle @@ -14719,8 +15341,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const int k = tensor->ne[0]; transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); transpose_2d_as_8b(backend_ctx, extra->hm, hm, extra->size_hm, m, k/8, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sc, sc, extra->size_sc, m, 3*k/(int)blck, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, d, extra->size_d, m, k/(int)blck, true, true); + transpose_2d_as_32b(backend_ctx, extra->sc, sc, extra->size_sc, m, 3*k/(int)blck, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/(int)blck, true, true); } #endif cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_q3_k_noshuffle @@ -14742,12 +15364,12 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, return; } - if (tensor->type == GGML_TYPE_IQ1_M) { + if (tensor->type == GGML_TYPE_IQ3_S) { if (tensor->view_src) { offset += tensor->view_offs; tensor = tensor->view_src; } - ggml_tensor_extra_cl_iq1_m * extra = (ggml_tensor_extra_cl_iq1_m *)tensor->extra; + ggml_tensor_extra_cl_iq3_s * extra = (ggml_tensor_extra_cl_iq3_s *)tensor->extra; const size_t blck = (size_t)ggml_blck_size(tensor->type); @@ -14757,29 +15379,40 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; cl_ulong nb_arg = (cl_ulong)n_blk; - cl_mem qs = extra->qs, qh = extra->qh, sc = extra->sc; + cl_mem qs = extra->qs, sg = extra->sg, qh = extra->qh, sc = extra->sc, d = extra->d; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - ggml_cl_buffer buf_qs, buf_qh, buf_sc; + ggml_cl_buffer buf_qs, buf_sg, buf_qh, buf_sc, buf_d; if (extra->noshuffle) { buf_qs.allocate(context, extra->size_qs); - qs = buf_qs.buffer; + buf_sg.allocate(context, extra->size_sg); buf_qh.allocate(context, extra->size_qh); - qh = buf_qh.buffer; buf_sc.allocate(context, extra->size_sc); + buf_d.allocate(context, extra->size_d); + + qs = buf_qs.buffer; + sg = buf_sg.buffer; + qh = buf_qh.buffer; sc = buf_sc.buffer; + d = buf_d.buffer; + const int m = tensor->ne[1]; const int k = tensor->ne[0]; - transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/8, true, true); - transpose_2d_as_8b(backend_ctx, extra->qh, qh, extra->size_qh, m, k/16, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->sc, sc, extra->size_sc, m, k/64, true, true); + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); + transpose_2d_as_8b(backend_ctx, extra->sg, sg, extra->size_sg, m, k/8, true, true); + transpose_2d_as_8b(backend_ctx, extra->qh, qh, extra->size_qh, m, k/32, true, true); + transpose_2d_as_8b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/64, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/256, true, true); } #endif - cl_kernel kernel = backend_ctx->kernel_restore_block_iq1_m; + + cl_kernel kernel = backend_ctx->kernel_restore_block_iq3_s; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &sc)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sg)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb_arg)); size_t gws[] = { (n_blk + 63) / 64 * 64 }; size_t lws[] = { 64 }; @@ -14791,12 +15424,113 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, return; } - if (tensor->type == GGML_TYPE_IQ1_S) { + if (tensor->type == GGML_TYPE_IQ3_XXS) { if (tensor->view_src) { offset += tensor->view_offs; tensor = tensor->view_src; } - ggml_tensor_extra_cl_iq1_s * extra = (ggml_tensor_extra_cl_iq1_s *)tensor->extra; + ggml_tensor_extra_cl_iq3_xxs * extra = (ggml_tensor_extra_cl_iq3_xxs *)tensor->extra; + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + + cl_mem qs = extra->qs, sas = extra->sas, d = extra->d; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_sas, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + buf_sas.allocate(context, extra->size_sas); + buf_d.allocate(context, extra->size_d); + + qs = buf_qs.buffer; + sas = buf_sas.buffer; + d = buf_d.buffer; + + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); + transpose_2d_as_32b(backend_ctx, extra->sas, sas, extra->size_sas, m, k/32, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/256, true, true); + } +#endif + + cl_kernel kernel = backend_ctx->kernel_restore_block_iq3_xxs; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sas)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } + + if (tensor->type == GGML_TYPE_IQ1_M) { + if (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; + } + ggml_tensor_extra_cl_iq1_m * extra = (ggml_tensor_extra_cl_iq1_m *)tensor->extra; + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + + cl_mem qs = extra->qs, qh = extra->qh, sc = extra->sc; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_qh, buf_sc; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + qs = buf_qs.buffer; + buf_qh.allocate(context, extra->size_qh); + qh = buf_qh.buffer; + buf_sc.allocate(context, extra->size_sc); + sc = buf_sc.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/8, true, true); + transpose_2d_as_8b(backend_ctx, extra->qh, qh, extra->size_qh, m, k/16, true, true); + transpose_2d_as_16b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/64, true, true); + } +#endif + cl_kernel kernel = backend_ctx->kernel_restore_block_iq1_m; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } + + if (tensor->type == GGML_TYPE_IQ1_S) { + if (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; + } + ggml_tensor_extra_cl_iq1_s * extra = (ggml_tensor_extra_cl_iq1_s *)tensor->extra; const size_t blck = (size_t)ggml_blck_size(tensor->type); @@ -14819,8 +15553,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const int m = tensor->ne[1]; const int k = tensor->ne[0]; transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/8, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->qh, qh, extra->size_qh, m, k/32, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, d, extra->size_d, m, k/(int)blck, true, true); + transpose_2d_as_16b(backend_ctx, extra->qh, qh, extra->size_qh, m, k/32, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/(int)blck, true, true); } #endif cl_kernel kernel = backend_ctx->kernel_restore_block_iq1_s; @@ -14875,7 +15609,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, transpose_2d_as_8b(backend_ctx, extra->sg, sg, extra->size_sg, m, k/8, true, true); transpose_2d_as_8b(backend_ctx, extra->qh, qh, extra->size_qh, m, k/32, true, true); transpose_2d_as_8b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/32, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, d, extra->size_d, m, k/(int)blck, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/(int)blck, true, true); } #endif cl_kernel kernel = backend_ctx->kernel_restore_block_iq2_s; @@ -14924,9 +15658,9 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, d = buf_d.buffer; const int m = tensor->ne[1]; const int k = tensor->ne[0]; - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->qs, qs, extra->size_qs, m, k/8, true, true); + transpose_2d_as_16b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/8, true, true); transpose_2d_as_8b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/32, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, d, extra->size_d, m, k/(int)blck, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/(int)blck, true, true); } #endif cl_kernel kernel = backend_ctx->kernel_restore_block_iq2_xs; @@ -14974,8 +15708,8 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const int m = tensor->ne[1]; const int k = tensor->ne[0]; transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/8, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sas, sas, extra->size_sas, m, k/32, true, true); - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, d, extra->size_d, m, k/(int)blck, true, true); + transpose_2d_as_32b(backend_ctx, extra->sas, sas, extra->size_sas, m, k/32, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/(int)blck, true, true); } #endif cl_kernel kernel = backend_ctx->kernel_restore_block_iq2_xxs; @@ -27671,6 +28405,124 @@ static bool ggml_cl_mul_mat_q3_K_dp4a(ggml_backend_t backend, const ggml_tensor return true; } +static bool ggml_cl_mul_mat_iq3_s_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq3_s * extra0 = (ggml_tensor_extra_cl_iq3_s *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 8 ? 8 : 32; + cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq3_s_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_iq3_s_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_IQ3_S_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } + + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sg)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 11, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} + +static bool ggml_cl_mul_mat_iq3_xxs_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq3_xxs * extra0 = (ggml_tensor_extra_cl_iq3_xxs *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 8 ? 8 : 32; + cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_IQ3_XXS_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } + + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sas)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} + static bool ggml_cl_mul_mat_iq1_m_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl_iq1_m * extra0 = (ggml_tensor_extra_cl_iq1_m *)src0->extra; @@ -28166,9 +29018,9 @@ static void ggml_cl_mul_mat_q3_k_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clReleaseMemObject(b_sub_buf)); } -static void ggml_cl_mul_mat_iq1_m_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_iq3_s_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl_iq1_m * extra0 = (ggml_tensor_extra_cl_iq1_m *)src0->extra; + ggml_tensor_extra_cl_iq3_s * extra0 = (ggml_tensor_extra_cl_iq3_s *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; @@ -28179,29 +29031,31 @@ static void ggml_cl_mul_mat_iq1_m_f32_adreno(ggml_backend_t backend, const ggml_ const int k = src0->ne[0]; if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { - cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq1_m_f32; + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq3_s_f32; const int stride_d = dst->ne[0]; const int stride_b = src1->nb[1]/sizeof(float); - CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq1_s_grid_image)); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq3_s_grid_image)); CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->qs)); - CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->qh)); - CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->sc)); - CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); - CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); - CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); - CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); - const size_t nsg = backend_ctx->iq1_m_mv_nsg; + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sg)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 12, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 13, sizeof(int), &stride_d)); + const size_t nsg = backend_ctx->iq3_s_mv_nsg; size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; size_t local[] = { 64, nsg, 1 }; backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); return; } - if (ggml_cl_mul_mat_iq1_m_dp4a(backend, src0, src1, dst)) { + if (ggml_cl_mul_mat_iq3_s_dp4a(backend, src0, src1, dst)) { return; } @@ -28245,17 +29099,19 @@ static void ggml_cl_mul_mat_iq1_m_f32_adreno(ggml_backend_t backend, const ggml_ size_t t_local[] = { 1, 16 }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); - kernel = backend_ctx->kernel_gemm_noshuffle_iq1_m_f32; + kernel = backend_ctx->kernel_gemm_noshuffle_iq3_s_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->sc)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sg)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &n)); size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; size_t local[] = { 1, 128, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); @@ -28266,9 +29122,209 @@ static void ggml_cl_mul_mat_iq1_m_f32_adreno(ggml_backend_t backend, const ggml_ CL_CHECK(clReleaseMemObject(b_sub_buf)); } -static void ggml_cl_mul_mat_iq1_s_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_iq3_xxs_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl_iq1_s * extra0 = (ggml_tensor_extra_cl_iq1_s *)src0->extra; + ggml_tensor_extra_cl_iq3_xxs * extra0 = (ggml_tensor_extra_cl_iq3_xxs *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq3_xxs_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq3_xxs_grid_image)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sas)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); + const size_t nsg = backend_ctx->iq3_xxs_mv_nsg; + size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } + + if (ggml_cl_mul_mat_iq3_xxs_dp4a(backend, src0, src1, dst)) { + return; + } + + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_iq3_xxs_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sas)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} + +static void ggml_cl_mul_mat_iq1_m_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq1_m * extra0 = (ggml_tensor_extra_cl_iq1_m *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq1_m_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq1_s_grid_image)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); + const size_t nsg = backend_ctx->iq1_m_mv_nsg; + size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } + + if (ggml_cl_mul_mat_iq1_m_dp4a(backend, src0, src1, dst)) { + return; + } + + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_iq1_m_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} + +static void ggml_cl_mul_mat_iq1_s_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq1_s * extra0 = (ggml_tensor_extra_cl_iq1_s *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; @@ -28792,6 +29848,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co ggml_tensor_extra_cl_iq2_xxs * extra0_iq2_xxs = (ggml_tensor_extra_cl_iq2_xxs *)soa0_src->extra; ggml_tensor_extra_cl_iq1_m * extra0_iq1_m = (ggml_tensor_extra_cl_iq1_m *)soa0_src->extra; ggml_tensor_extra_cl_iq1_s * extra0_iq1_s = (ggml_tensor_extra_cl_iq1_s *)soa0_src->extra; + ggml_tensor_extra_cl_iq3_s * extra0_iq3_s = (ggml_tensor_extra_cl_iq3_s *)soa0_src->extra; + ggml_tensor_extra_cl_iq3_xxs * extra0_iq3_xxs = (ggml_tensor_extra_cl_iq3_xxs *)soa0_src->extra; ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra; ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra; ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra; @@ -29205,6 +30263,18 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co return; } + // iq3_s x fp32 + if (src0t == GGML_TYPE_IQ3_S && src1t == GGML_TYPE_F32 && extra0_iq3_s->noshuffle) { + ggml_cl_mul_mat_iq3_s_f32_adreno(backend, src0, src1, dst); + return; + } + + // iq3_xxs x fp32 + if (src0t == GGML_TYPE_IQ3_XXS && src1t == GGML_TYPE_F32 && extra0_iq3_xxs->noshuffle) { + ggml_cl_mul_mat_iq3_xxs_f32_adreno(backend, src0, src1, dst); + return; + } + // iq1_m x fp32 if (src0t == GGML_TYPE_IQ1_M && src1t == GGML_TYPE_F32 && extra0_iq1_m->noshuffle) { ggml_cl_mul_mat_iq1_m_f32_adreno(backend, src0, src1, dst); @@ -29873,6 +30943,96 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } + case GGML_TYPE_IQ3_S: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_iq3_s_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq3_s->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq3_s->sg)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq3_s->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq3_s->sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_iq3_s->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_IQ3_XXS: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_iq3_xxs_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq3_xxs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq3_xxs->sas)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq3_xxs->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } case GGML_TYPE_IQ1_M: { if (ne11 < 32) { break; @@ -31200,6 +32360,134 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co #endif // GGML_OPENCL_SOA_Q break; } + case GGML_TYPE_IQ3_XXS: { + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq3_xxs_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq3_xxs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq3_xxs->sas)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq3_xxs->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_iq3_xxs_f32; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_IQ3_S: { + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq3_s_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq3_s->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq3_s->sg)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq3_s->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq3_s->sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_iq3_s->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_iq3_s_f32; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } case GGML_TYPE_IQ1_M: { if (backend_ctx->gpu_family == INTEL) { nth0 = 16; @@ -31825,7 +33113,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co src0t == GGML_TYPE_Q1_0 || src0t == GGML_TYPE_Q2_0 || src0t == GGML_TYPE_IQ4_NL || - src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K || src0t == GGML_TYPE_IQ2_S || src0t == GGML_TYPE_IQ2_XS || src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ1_M) { + src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K || src0t == GGML_TYPE_IQ2_S || src0t == GGML_TYPE_IQ2_XS || src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ1_M || src0t == GGML_TYPE_IQ3_S || src0t == GGML_TYPE_IQ3_XXS) { // Each SIMD group produces N_DST values in the result. Assuming each // workgroup has N_SIMDGROUP SIMD groups, then each workgroup will // produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl index 582d2e59ee5f..143f7c24d1df 100644 --- a/ggml/src/ggml-opencl/kernels/cvt.cl +++ b/ggml/src/ggml-opencl/kernels/cvt.cl @@ -3178,3 +3178,108 @@ kernel void kernel_restore_block_iq1_m( dst[i].scales[2*j + 1] = (uchar)(sc >> 8); } } + +struct block_iq3_s { + half d; + uchar qs[QK_K/4]; + uchar qh[QK_K/32]; + uchar signs[QK_K/8]; + uchar scales[QK_K/64]; +}; + +kernel void kernel_convert_block_iq3_s( + global struct block_iq3_s * src0, + global uchar * dst_qs, + global uchar * dst_sg, + global uchar * dst_qh, + global uchar * dst_sc, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + for (int j = 0; j < 64; ++j) { + dst_qs[64*i + j] = src0[i].qs[j]; + } + for (int j = 0; j < 32; ++j) { + dst_sg[32*i + j] = src0[i].signs[j]; + } + for (int j = 0; j < 8; ++j) { + dst_qh[8*i + j] = src0[i].qh[j]; + } + for (int j = 0; j < 4; ++j) { + dst_sc[4*i + j] = src0[i].scales[j]; + } +} + +kernel void kernel_restore_block_iq3_s( + global uchar * src_qs, + global uchar * src_sg, + global uchar * src_qh, + global uchar * src_sc, + global half * src_d, + global struct block_iq3_s * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + for (int j = 0; j < 64; ++j) { + dst[i].qs[j] = src_qs[64*i + j]; + } + for (int j = 0; j < 32; ++j) { + dst[i].signs[j] = src_sg[32*i + j]; + } + for (int j = 0; j < 8; ++j) { + dst[i].qh[j] = src_qh[8*i + j]; + } + for (int j = 0; j < 4; ++j) { + dst[i].scales[j] = src_sc[4*i + j]; + } +} + +struct block_iq3_xxs { + half d; + uchar qs[3*QK_K/8]; +}; + +kernel void kernel_convert_block_iq3_xxs( + global struct block_iq3_xxs * src0, + global uchar * dst_qs, + global uint * dst_sas, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + for (int j = 0; j < 64; ++j) { + dst_qs[64*i + j] = src0[i].qs[j]; + } + for (int j = 0; j < 8; ++j) { + global ushort * q = (global ushort *)(src0[i].qs + 64 + 4*j); + dst_sas[8*i + j] = (uint)q[0] | ((uint)q[1] << 16); + } +} + +kernel void kernel_restore_block_iq3_xxs( + global uchar * src_qs, + global uint * src_sas, + global half * src_d, + global struct block_iq3_xxs * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + for (int j = 0; j < 64; ++j) { + dst[i].qs[j] = src_qs[64*i + j]; + } + for (int j = 0; j < 8; ++j) { + global ushort * q = (global ushort *)(dst[i].qs + 64 + 4*j); + uint sas = src_sas[8*i + j]; + q[0] = (ushort)sas; + q[1] = (ushort)(sas >> 16); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_f32.cl new file mode 100644 index 000000000000..1a15ff86cdb2 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_f32.cl @@ -0,0 +1,92 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_iq3_s_f32( + global const uchar * src0_qs, + global const uchar * src0_sg, + global const uchar * src0_qh, + global const uchar * src0_sc, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const uchar4 sc = vload4(0, src0_sc + row + (i / 64) * m); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const float4 scale_f = convert_float4(d) * (1.0f + 2.0f*convert_float4((sc >> (uchar4)(4*((i / 32) & 1))) & (uchar4)(15))); + const half4 scale = convert_half4(scale_f); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uint4 qs = convert_uint4(vload4(0, src0_qs + row + (ki / 4) * m)); + const uint4 qh = convert_uint4(vload4(0, src0_qh + row + (ki / 32) * m)); + const uchar4 sg = vload4(0, src0_sg + row + (ki / 8) * m); + const uint4 gi = qs | (((qh >> (uint4)(((ki / 4) & 7))) & (uint4)(1)) << (uint4)(8)); + const uint4 grid = (uint4)(iq3_s_grid[gi.s0], iq3_s_grid[gi.s1], + iq3_s_grid[gi.s2], iq3_s_grid[gi.s3]); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255)); + const int4 signs = convert_int4((sg >> (uchar4)((ki & 7) + j)) & (uchar4)(1)); + const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_q8_1_dp4a.cl new file mode 100644 index 000000000000..43cb38f72067 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_q8_1_dp4a.cl @@ -0,0 +1,154 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint iq3_s_pack(uint gv, uint sg, uint base) { + int v0 = (int)((gv >> 0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; } + int v1 = (int)((gv >> 8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; } + int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; } + int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; } + return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8) + | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24); +} + +#define IQ3_S_GEMM_GRID(i) iq3_s_grid[(i)] + +inline uint iq3_s_load(__global const uchar * qs, uint qsb, uint qhb, uint m, + uint u, uint sgv, uint base) { + const uint gidx = (uint)qs[qsb + u * m] | (((qhb >> u) & 1u) << 8); + return iq3_s_pack(IQ3_S_GEMM_GRID(gidx), sgv, base); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq3_s_q8_1_dp4a( + __global const uchar * src0_qs, + __global const uchar * src0_sg, + __global const uchar * src0_qh, + __global const uchar * src0_sc, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const uint ib = sub >> 3; + const uint sb = sub & 7u; + + const uint scb = (uint)src0_sc[rrow + (ib*(QK_K/64) + (sb >> 1)) * (uint)m]; + const uint nib = (sb & 1u) ? (scb >> 4) : (scb & 0xFu); + const float d_w = (float)src0_d[rrow + ib * (uint)m] * (float)(1 + 2*nib); + + const uint qhb = (uint)src0_qh[rrow + (ib*(QK_K/32) + sb) * (uint)m]; + const uint qsb = rrow + (ib*(QK_K/4) + 8*sb) * (uint)m; + const uint sgb = rrow + (ib*(QK_K/8) + 4*sb) * (uint)m; + + const uint sg0 = (uint)src0_sg[sgb + 0 * (uint)m]; + const uint sg1 = (uint)src0_sg[sgb + 1 * (uint)m]; + const uint sg2 = (uint)src0_sg[sgb + 2 * (uint)m]; + const uint sg3 = (uint)src0_sg[sgb + 3 * (uint)m]; + + uint8 qw; + qw.s0 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 0u, sg0, 0u); + qw.s1 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 1u, sg0, 4u); + qw.s2 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 2u, sg1, 0u); + qw.s3 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 3u, sg1, 4u); + qw.s4 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 4u, sg2, 0u); + qw.s5 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 5u, sg2, 4u); + qw.s6 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 6u, sg3, 0u); + qw.s7 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 7u, sg3, 4u); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1])) + rf.s0 = (float)TDOT4(b+0); rf.s1 = (float)TDOT4(b+1); + rf.s2 = (float)TDOT4(b+2); rf.s3 = (float)TDOT4(b+3); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef TDOT4 +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_f32.cl new file mode 100644 index 000000000000..af58e850cc2e --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_f32.cl @@ -0,0 +1,89 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_iq3_xxs_f32( + global const uchar * src0_qs, + global const uint * src0_sas, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const uint4 sas = vload4(0, src0_sas + row + (i / 32) * m); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const float4 scale_f = convert_float4(d) * 0.5f * (0.5f + convert_float4(sas >> (uint4)(28))); + const half4 scale = convert_half4(scale_f); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uint4 gi = convert_uint4(vload4(0, src0_qs + row + (ki / 4) * m)); + const uint4 code = (sas >> (uint4)(7*((ki / 8) & 3))) & (uint4)(127); + const uint4 sg = code | ((popcount(code) & (uint4)(1)) << (uint4)(7)); + const uint4 grid = (uint4)(iq3_xxs_grid[gi.s0], iq3_xxs_grid[gi.s1], + iq3_xxs_grid[gi.s2], iq3_xxs_grid[gi.s3]); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255)); + const int4 signs = convert_int4((sg >> (uint4)((ki & 7) + j)) & (uint4)(1)); + const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl new file mode 100644 index 000000000000..2ade6e2cf4c2 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl @@ -0,0 +1,137 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint iq3_xxs_pack(uint gv, uint sg, uint base) { + int v0 = (int)((gv >> 0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; } + int v1 = (int)((gv >> 8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; } + int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; } + int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; } + return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8) + | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a( + __global const uchar * src0_qs, + __global const uint * src0_sas, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const uint ib = sub >> 3; + const uint aux = src0_sas[rrow + sub*(uint)m]; + const float d_w = (float)src0_d[rrow + ib*(uint)m] * (0.5f + (float)(aux >> 28)) * 0.5f; + const uint sg0 = iq3_xxs_signs( aux & 127u); + const uint sg1 = iq3_xxs_signs((aux >> 7) & 127u); + const uint sg2 = iq3_xxs_signs((aux >> 14) & 127u); + const uint sg3 = iq3_xxs_signs((aux >> 21) & 127u); + const uint qsb = rrow + sub*8u*(uint)m; + + uint8 qw; + qw.s0 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 0u*(uint)m]], sg0, 0u); + qw.s1 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 1u*(uint)m]], sg0, 4u); + qw.s2 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 2u*(uint)m]], sg1, 0u); + qw.s3 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 3u*(uint)m]], sg1, 4u); + qw.s4 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 4u*(uint)m]], sg2, 0u); + qw.s5 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 5u*(uint)m]], sg2, 4u); + qw.s6 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 6u*(uint)m]], sg3, 0u); + qw.s7 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 7u*(uint)m]], sg3, 4u); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1])) + rf.s0 = (float)TDOT4(b+0); rf.s1 = (float)TDOT4(b+1); + rf.s2 = (float)TDOT4(b+2); rf.s3 = (float)TDOT4(b+3); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef TDOT4 +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_s_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_s_f32.cl new file mode 100644 index 000000000000..e94b84a86c05 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_s_f32.cl @@ -0,0 +1,128 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef IQ3_S_MV_NSG +#define IQ3_S_MV_NSG 4 +#endif + +inline float4 iq3_s_vals(uint gv, uint sg, uint base) { + + const uint s = sg >> base; + const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u; + return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn); +} + +kernel void kernel_gemv_noshuffle_iq3_s_f32( + __read_only image1d_buffer_t grid_img, + global const uchar * src0_qs, + global const uchar * src0_sg, + global const uchar * src0_qh, + global const uchar * src0_sc, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne10, + int ne0 +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + +#define IQ3_S_GRID(i) (read_imageui(grid_img, (int)(i)).x) + + const uint mh = m >> 1; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 1; + + float sumf = 0.f; + float sumf1 = 0.f; + + if (j < mh) { + global const ushort * qsu = (global const ushort *)src0_qs; + global const ushort * qhu = (global const ushort *)src0_qh; + global const ushort * sgu = (global const ushort *)src0_sg; + global const ushort * scu = (global const ushort *)src0_sc; + + for (uint ib = sgi; ib < nsb; ib += IQ3_S_MV_NSG) { + const half2 dh = vload2(j + ib * mh, src0_d); + const float d0 = (float)dh.s0; + const float d1 = (float)dh.s1; + + const uint scbase = j + ib * 4u * mh; + ushort sc4[4]; + sc4[0] = scu[scbase + 0u * mh]; + sc4[1] = scu[scbase + 1u * mh]; + sc4[2] = scu[scbase + 2u * mh]; + sc4[3] = scu[scbase + 3u * mh]; + + float acc0 = 0.f, acc1 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint scv = (uint)sc4[sb >> 1]; + const uint nib0 = (sb & 1u) ? ((scv >> 4) & 0xFu) : ( scv & 0xFu); + const uint nib1 = (sb & 1u) ? ((scv >> 12) & 0xFu) : ((scv >> 8) & 0xFu); + + const uint qhv = (uint)qhu[j + (ib * 8u + sb) * mh]; + const uint qh0 = qhv & 0xFFu; + const uint qh1 = qhv >> 8; + + const uint grp = ib * 64u + sb * 8u; + const uint qsb = j + grp * mh; + const uint sgb = j + (ib * 32u + sb * 4u) * mh; + + float a0 = 0.f, a1 = 0.f; + for (uint u = 0; u < 8u; ++u) { + const uint qsv = (uint)qsu[qsb + u * mh]; + const uint sgv = (uint)sgu[sgb + (u >> 1) * mh]; + const uint g0 = ( qsv & 0xFFu) | (((qh0 >> u) & 1u) << 8); + const uint g1 = ((qsv >> 8) & 0xFFu) | (((qh1 >> u) & 1u) << 8); + const uint base = (u & 1u) * 4u; + const float4 yv = vload4(grp + u, y); + const uint gw0 = IQ3_S_GRID(g0); + const uint gw1 = IQ3_S_GRID(g1); + a0 += dot(yv, iq3_s_vals(gw0, sgv & 0xFFu, base)); + a1 += dot(yv, iq3_s_vals(gw1, (sgv >> 8) & 0xFFu, base)); + } + acc0 += (float)(1u + 2u * nib0) * a0; + acc1 += (float)(1u + 2u * nib1) * a1; + } + sumf += d0 * acc0; + sumf1 += d1 * acc1; + } + } + +#if IQ3_S_MV_NSG > 1 + __local float2 part[IQ3_S_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < IQ3_S_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + } +#endif + + if (j < mh) { + vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } +} + +kernel void kernel_iq3_s_grid_export(global uint * out) { + out[get_global_id(0)] = iq3_s_grid[get_global_id(0)]; +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_xxs_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_xxs_f32.cl new file mode 100644 index 000000000000..c6a5dbeac347 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_xxs_f32.cl @@ -0,0 +1,120 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef IQ3_XXS_MV_NSG +#define IQ3_XXS_MV_NSG 4 +#endif + +inline float4 iq3_xxs_vals(uint gv, uint sgv, uint base) { + const uint s = sgv >> base; + const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u; + return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn); +} + +kernel void kernel_gemv_noshuffle_iq3_xxs_f32( + __read_only image1d_buffer_t grid_img, + global const uchar * src0_qs, + global const uint * src0_sas, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne10, + int ne0 +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + +#define IQ3_XXS_GRID(i) (read_imageui(grid_img, (int)(i)).x) + + const uint mh = m >> 1; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 1; + + float sumf = 0.f; + float sumf1 = 0.f; + + if (j < mh) { + global const ushort * qsu = (global const ushort *)src0_qs; + + for (uint ib = sgi; ib < nsb; ib += IQ3_XXS_MV_NSG) { + const half2 dh = vload2(j + ib * mh, src0_d); + const float d0 = (float)dh.s0; + const float d1 = (float)dh.s1; + + float acc0 = 0.f, acc1 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint sub = ib * 8u + sb; + const uint2 aux = vload2(j + sub * mh, src0_sas); + + const uint grp = sub * 8u; + const uint qsb = j + grp * mh; + + float a0 = 0.f, a1 = 0.f; + for (uint u = 0; u < 4u; ++u) { + const uint qsv0 = (uint)qsu[qsb + (2u*u)*mh]; + const uint qsv1 = (uint)qsu[qsb + (2u*u + 1u)*mh]; + const uint sg0 = iq3_xxs_signs((aux.s0 >> (7u * u)) & 127u); + const uint sg1 = iq3_xxs_signs((aux.s1 >> (7u * u)) & 127u); + + const uint g00 = qsv0 & 0xFFu; + const uint g01 = qsv1 & 0xFFu; + const uint g10 = (qsv0 >> 8) & 0xFFu; + const uint g11 = (qsv1 >> 8) & 0xFFu; + + const float4 yl = vload4(sub * 8u + u * 2u + 0u, y); + const float4 yh = vload4(sub * 8u + u * 2u + 1u, y); + + a0 += dot(yl, iq3_xxs_vals(IQ3_XXS_GRID(g00), sg0, 0u)) + + dot(yh, iq3_xxs_vals(IQ3_XXS_GRID(g01), sg0, 4u)); + a1 += dot(yl, iq3_xxs_vals(IQ3_XXS_GRID(g10), sg1, 0u)) + + dot(yh, iq3_xxs_vals(IQ3_XXS_GRID(g11), sg1, 4u)); + } + acc0 += (0.5f + (float)(aux.s0 >> 28)) * a0; + acc1 += (0.5f + (float)(aux.s1 >> 28)) * a1; + } + sumf += d0 * 0.5f * acc0; + sumf1 += d1 * 0.5f * acc1; + } + } + +#if IQ3_XXS_MV_NSG > 1 + __local float2 part[IQ3_XXS_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < IQ3_XXS_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + } +#endif + + if (j < mh) { + vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } +#undef IQ3_XXS_GRID +} + +kernel void kernel_iq3_xxs_grid_export(global uint * out) { + const uint i = get_global_id(0); + if (i < 256u) { + out[i] = iq3_xxs_grid[i]; + } +} diff --git a/ggml/src/ggml-opencl/kernels/iq3_s_grid.cl b/ggml/src/ggml-opencl/kernels/iq3_s_grid.cl new file mode 100644 index 000000000000..7ffd2c4fe5f6 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/iq3_s_grid.cl @@ -0,0 +1,66 @@ +constant uint iq3_s_grid[512] = { + 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305, + 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905, + 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09, + 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b, + 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b, + 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d, + 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03, + 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505, + 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03, + 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901, + 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d, + 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303, + 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501, + 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105, + 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505, + 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101, + 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707, + 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b, + 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01, + 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f, + 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305, + 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103, + 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509, + 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503, + 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b, + 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f, + 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f, + 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f, + 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109, + 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f, + 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509, + 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501, + 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303, + 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f, + 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907, + 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703, + 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03, + 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01, + 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01, + 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903, + 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505, + 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b, + 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107, + 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509, + 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303, + 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103, + 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05, + 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b, + 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f, + 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701, + 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909, + 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305, + 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d, + 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b, + 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d, + 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307, + 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09, + 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309, + 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709, + 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f, + 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303, + 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503, + 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b, + 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101 +}; diff --git a/ggml/src/ggml-opencl/kernels/iq3_xxs_grid.cl b/ggml/src/ggml-opencl/kernels/iq3_xxs_grid.cl new file mode 100644 index 000000000000..4c79ddf1adcc --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/iq3_xxs_grid.cl @@ -0,0 +1,38 @@ +constant uint iq3_xxs_grid[256] = { + 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, + 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, + 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, + 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e, + 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, + 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c, + 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34, + 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c, + 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c, + 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04, + 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, + 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, + 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434, + 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, + 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e, + 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24, + 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24, + 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c, + 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c, + 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, + 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, + 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e, + 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, + 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c, + 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c, + 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14, + 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c, + 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c, + 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, + 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, + 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c, + 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04 +}; + +inline uint iq3_xxs_signs(uint code) { + return code | ((popcount(code) & 1u) << 7); +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq3_s_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq3_s_f32_l4_lm.cl new file mode 100644 index 000000000000..e75c296ac012 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq3_s_f32_l4_lm.cl @@ -0,0 +1,193 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_iq3_s { + half d; + uchar qs[QK_K/4]; + uchar qh[QK_K/32]; + uchar signs[QK_K/8]; + uchar scales[QK_K/64]; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#ifdef INTEL_GPU +#define TM 8 +#define TN 8 +#else +#define TM 4 +#define TN 8 +#endif + +kernel void kernel_mul_mm_iq3_s_f32_l4_lm( + global uchar * src0_qs, + global uchar * src0_sg, + global uchar * src0_qh, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one iq3_s super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_iq3_s); + int e = idx % QK_K; + + int ib32 = e >> 5; + int rem = e & 31; + int lg = rem >> 3; + int j0 = rem & 7; + + uchar scb = src0_sc[4*ib + ib32/2]; + float db = (float)src0_d[ib] * (float)(1 + 2*((scb >> (4*(ib32 & 1))) & 15)); + int which = j0 >> 2; + uint gi = (uint)src0_qs[64*ib + 8*ib32 + 2*lg + which] + | (((uint)src0_qh[8*ib + ib32] << (8 - 2*lg - which)) & 256); + uchar sg = src0_sg[32*ib + 4*ib32 + lg]; + uint g = iq3_s_grid[gi]; + + float4 v1; + v1.s0 = db * (float)((g >> 0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f); + v1.s1 = db * (float)((g >> 8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f); + v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f); + v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f); + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq3_xxs_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq3_xxs_f32_l4_lm.cl new file mode 100644 index 000000000000..3c50d62303c6 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq3_xxs_f32_l4_lm.cl @@ -0,0 +1,187 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_iq3_xxs { + half d; + uchar qs[3*QK_K/8]; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#ifdef INTEL_GPU +#define TM 8 +#define TN 8 +#else +#define TM 4 +#define TN 8 +#endif + +kernel void kernel_mul_mm_iq3_xxs_f32_l4_lm( + global uchar * src0_qs, + global uint * src0_sas, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one iq3_xxs super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_iq3_xxs); + int e = idx % QK_K; + + int ib32 = e >> 5; + int rem = e & 31; + int lg = rem >> 3; + int j0 = rem & 7; + + uint aux = src0_sas[8*ib + ib32]; + float db = (float)src0_d[ib] * (0.5f + (float)(aux >> 28)) * 0.5f; + + uint gi = src0_qs[64*ib + 8*ib32 + 2*lg + (j0 >> 2)]; + uint sg = iq3_xxs_signs((aux >> (7*lg)) & 127u); + uint g = iq3_xxs_grid[gi]; + + float4 v1; + v1.s0 = db * (float)((g >> 0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f); + v1.s1 = db * (float)((g >> 8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f); + v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f); + v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f); + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32.cl new file mode 100644 index 000000000000..0d9b67b4d1b6 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32.cl @@ -0,0 +1,146 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 +#define IQ3_S_N_SCALE (QK_K/64) + +typedef struct { + half d; + uchar qs[QK_K/4]; + uchar qh[QK_K/32]; + uchar signs[QK_K/8]; + uchar scales[IQ3_S_N_SCALE]; +} block_iq3_s; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq3_s_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_iq3_s * x = (global block_iq3_s *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + global char * xrow = (global char *)(x + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + global block_iq3_s * xb = (global block_iq3_s *)(xrow + row*nb01); + + uchar nib = (it & 1) ? (xb->scales[it >> 1] >> 4) : (xb->scales[it >> 1] & 0xf); + float db = (float)xb->d * (float)(1 + 2*nib); + + uchar qhb = xb->qh[it]; + global uchar * qsb = xb->qs + 8*it; + global uchar * sgb = xb->signs + 4*it; + + float acc = 0.f; + for (int l = 0; l < 4; ++l) { + uint i1 = (uint)qsb[2*l+0] | (((uint)qhb << (8-2*l)) & 256); + uint i2 = (uint)qsb[2*l+1] | (((uint)qhb << (7-2*l)) & 256); + uint g1 = iq3_s_grid[i1]; + uint g2 = iq3_s_grid[i2]; + uchar sg = sgb[l]; + for (int j = 0; j < 4; ++j) { + acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f); + acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f); + } + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32_flat.cl new file mode 100644 index 000000000000..0208a53c3bca --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32_flat.cl @@ -0,0 +1,147 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 +#define IQ3_S_N_SCALE (QK_K/64) + +typedef struct { + half d; + uchar qs[QK_K/4]; + uchar qh[QK_K/32]; + uchar signs[QK_K/8]; + uchar scales[IQ3_S_N_SCALE]; +} block_iq3_s; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq3_s_f32_flat( + global uchar * src0_qs, + global uchar * src0_sg, + global uchar * src0_qh, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq3_s); + ulong row_stride = nb01/sizeof(block_iq3_s); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + const ulong b = offset_src0 + ib + row*row_stride; + + uchar nib = (it & 1) ? (src0_sc[4*b + (it >> 1)] >> 4) : (src0_sc[4*b + (it >> 1)] & 0xf); + float db = (float)src0_d[b] * (float)(1 + 2*nib); + + uchar qhb = src0_qh[8*b + it]; + global uchar * qsb = src0_qs + 64*b + 8*it; + global uchar * sgb = src0_sg + 32*b + 4*it; + + float acc = 0.f; + for (int l = 0; l < 4; ++l) { + uint i1 = (uint)qsb[2*l+0] | (((uint)qhb << (8-2*l)) & 256); + uint i2 = (uint)qsb[2*l+1] | (((uint)qhb << (7-2*l)) & 256); + uint g1 = iq3_s_grid[i1]; + uint g2 = iq3_s_grid[i2]; + uchar sg = sgb[l]; + for (int j = 0; j < 4; ++j) { + acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f); + acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f); + } + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32.cl new file mode 100644 index 000000000000..8355621d1ed8 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32.cl @@ -0,0 +1,145 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +// d, then 64 grid index bytes, then 8 uint32 of packed scale+signs +typedef struct { + half d; + uchar qs[3*QK_K/8]; +} block_iq3_xxs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq3_xxs_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_iq3_xxs * x = (global block_iq3_xxs *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + global char * xrow = (global char *)(x + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + global block_iq3_xxs * xb = (global block_iq3_xxs *)(xrow + row*nb01); + + global uchar * qs = xb->qs + 8*it; + + // Blocks are 98 bytes, so packed signs are only 2-byte aligned. + global ushort * sw = (global ushort *)(xb->qs + 64 + 4*it); + uint aux32 = (uint)sw[0] | ((uint)sw[1] << 16); + + float db = (float)xb->d * (0.5f + (float)(aux32 >> 28)) * 0.5f; + + float acc = 0.f; + for (int l = 0; l < 4; ++l) { + uchar signs = iq3_xxs_signs((aux32 >> (7*l)) & 127u); + uint g1 = iq3_xxs_grid[qs[2*l+0]]; + uint g2 = iq3_xxs_grid[qs[2*l+1]]; + + for (int j = 0; j < 4; ++j) { + float s1 = (signs & (1 << j)) ? -1.f : 1.f; + float s2 = (signs & (1 << (j+4))) ? -1.f : 1.f; + acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * s1; + acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * s2; + } + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32_flat.cl new file mode 100644 index 000000000000..974daf85915e --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32_flat.cl @@ -0,0 +1,142 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +// d, then 64 grid index bytes, then 8 uint32 of packed scale+signs +typedef struct { + half d; + uchar qs[3*QK_K/8]; +} block_iq3_xxs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq3_xxs_f32_flat( + global uchar * src0_qs, + global uint * src0_sas, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq3_xxs); + ulong row_stride = nb01/sizeof(block_iq3_xxs); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + const ulong b = offset_src0 + ib + row*row_stride; + + global uchar * qs = src0_qs + 64*b + 8*it; + + uint aux32 = src0_sas[8*b + it]; + + float db = (float)src0_d[b] * (0.5f + (float)(aux32 >> 28)) * 0.5f; + + float acc = 0.f; + for (int l = 0; l < 4; ++l) { + uchar signs = iq3_xxs_signs((aux32 >> (7*l)) & 127u); + uint g1 = iq3_xxs_grid[qs[2*l+0]]; + uint g2 = iq3_xxs_grid[qs[2*l+1]]; + + for (int j = 0; j < 4; ++j) { + float s1 = (signs & (1 << j)) ? -1.f : 1.f; + float s2 = (signs & (1 << (j+4))) ? -1.f : 1.f; + acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * s1; + acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * s2; + } + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} From dde879bc4f954b809892fd0d660515f42536394a Mon Sep 17 00:00:00 2001 From: Michael Taylor <162068037+mctylr-gh@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:46:43 -0400 Subject: [PATCH 12/36] sycl: Try PkgConfig for LevelZero includes / libs (#31076) --- ggml/src/ggml-sycl/CMakeLists.txt | 46 +++++++++++++++++++------------ 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index cba39a758336..35d36e583734 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -107,24 +107,36 @@ target_compile_options(ggml-sycl PRIVATE "-Wno-narrowing") message(STATUS "GGML_SYCL_SUPPORT_LEVEL_ZERO_API ${GGML_SYCL_SUPPORT_LEVEL_ZERO_API}") if (GGML_SYCL_SUPPORT_LEVEL_ZERO_API) - # Link against Level Zero loader for direct device memory allocation. - # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging - # in the xe kernel driver during multi-GPU inference. - find_path(LEVEL_ZERO_DEV_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include) - find_library(ZE_LOADER_LIB ze_loader HINTS ${ONEAPI_ROOT}/lib ${LEVEL_ZERO_V1_SDK_LIB_PATH} ENV LD_LIBRARY_PATH) - if(ZE_LOADER_LIB AND LEVEL_ZERO_DEV_INCLUDE_DIR) - target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB}) - target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API) - message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}") - message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_DEV_INCLUDE_DIR}") + find_package(PkgConfig) + if (PkgConfig_FOUND) + pkg_check_modules(LEVEL_ZERO level-zero) + if (LEVEL_ZERO_FOUND) + target_include_directories(ggml-sycl PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS}) + target_link_libraries(ggml-sycl PRIVATE ${LEVEL_ZERO_LINK_LIBRARIES}) + target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API) + message(STATUS "Level Zero loader found: ${LEVEL_ZERO_LINK_LIBRARIES}") + message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_INCLUDE_DIRS}") + endif() else() - message(WARNING "Level Zero loader or development headers not found, " - "Level Zero API support disabled. " - "Please install the Level Zero SDK/development package " - "to support Level Zero API features. " - "Level Zero API is not mandatory for SYCL backend, " - "but it is required by the special features for better " - "function & performance on Intel GPUs.") + # Link against Level Zero loader for direct device memory allocation. + # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging + # in the xe kernel driver during multi-GPU inference. + find_path(LEVEL_ZERO_DEV_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include) + find_library(ZE_LOADER_LIB ze_loader HINTS ${ONEAPI_ROOT}/lib ${LEVEL_ZERO_V1_SDK_LIB_PATH} ENV LD_LIBRARY_PATH) + if(ZE_LOADER_LIB AND LEVEL_ZERO_DEV_INCLUDE_DIR) + target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB}) + target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API) + message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}") + message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_DEV_INCLUDE_DIR}") + else() + message(WARNING "Level Zero loader or development headers not found, " + "Level Zero API support disabled. " + "Please install the Level Zero SDK/development package " + "to support Level Zero API features. " + "Level Zero API is not mandatory for SYCL backend, " + "but it is required by the special features for better " + "function & performance on Intel GPUs.") + endif() endif() endif() From 785322d17f2d51e5185cf996a1f4e46d87640841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sun, 1 Nov 2026 18:23:40 +0100 Subject: [PATCH 13/36] requirements : fix numpy dependencies (#31508) --- requirements/requirements-gguf_editor_gui.txt | 3 ++- requirements/requirements-server-bench.txt | 3 ++- requirements/requirements-tool_bench.txt | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/requirements/requirements-gguf_editor_gui.txt b/requirements/requirements-gguf_editor_gui.txt index 93fe087223eb..0618acf09919 100644 --- a/requirements/requirements-gguf_editor_gui.txt +++ b/requirements/requirements-gguf_editor_gui.txt @@ -1,3 +1,4 @@ -numpy~=2.2.6 +numpy~=2.2.6; python_version < "3.14" +numpy~=2.4.6; python_version >= "3.14" PySide6~=6.9.0 gguf>=0.17.0 diff --git a/requirements/requirements-server-bench.txt b/requirements/requirements-server-bench.txt index 0b065b1ea4c1..88a9cb2dc067 100644 --- a/requirements/requirements-server-bench.txt +++ b/requirements/requirements-server-bench.txt @@ -1,5 +1,6 @@ datasets~=4.8.0 matplotlib~=3.10.0 -numpy~=2.2.6 +numpy~=2.2.6; python_version < "3.14" +numpy~=2.4.6; python_version >= "3.14" requests~=2.32.3 tqdm~=4.67.1 diff --git a/requirements/requirements-tool_bench.txt b/requirements/requirements-tool_bench.txt index ba865115026d..e92186a6de61 100644 --- a/requirements/requirements-tool_bench.txt +++ b/requirements/requirements-tool_bench.txt @@ -1,7 +1,8 @@ aiohttp~=3.9.3 pytest~=8.3.3 matplotlib~=3.10.0 -numpy~=2.2.6 +numpy~=2.2.6; python_version < "3.14" +numpy~=2.4.6; python_version >= "3.14" openai~=2.14.0 pandas~=2.2.3 prometheus-client~=0.20.0 From 1c5ed702c9e540ac14cc8c78100ea3984bc0e227 Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 2 Nov 2026 08:36:04 +0100 Subject: [PATCH 14/36] webgpu : fix CI by keeping scratch alloc sizes monotonic in batch length (#31422) * webgpu: keep scratch alloc sizes monotonic in the batch length The flash attention vec path and the MMVQ path append a scratch area after dst, but get_alloc_size only accounted for it when the current batch takes that path. The size was therefore not monotonic in the batch length: a graph reserved at n_ubatch (tiled path, no scratch) or at one token did not cover batches of 2 to 19 query rows, and GGML_SCHED_NO_REALLOC builds aborted on an unexpected reallocation. Pass the row count to both path predicates and size the scratch with the row count clamped to the longest batch the short path serves. * nit: align the mmvq pipeline key with clang-format 22 * webgpu: address review @yomaytk, name the mat-vec row threshold --- .../ggml-webgpu/ggml-webgpu-shader-lib.hpp | 18 +++++----- ggml/src/ggml-webgpu/ggml-webgpu.cpp | 34 ++++++++++++------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp index e01a417704e5..1e63a9fac052 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp @@ -19,6 +19,7 @@ #define GGML_WEBGPU_I32_SIZE_BYTES 4 #define GGML_WEBGPU_FLASH_ATTN_PREFERRED_KV_SG_TILES 8u #define GGML_WEBGPU_FLASH_ATTN_VEC_MAX_SEQ_LEN 20u +#define GGML_WEBGPU_MUL_MAT_VEC_MAX_ROWS 4 #define GGML_WEBGPU_FLASH_ATTN_VEC_MAX_KV_TILE 32u #define GGML_WEBGPU_FLASH_ATTN_TILE_MAX_KV_TILE 64u #define GGML_WEBGPU_FLASH_ATTN_PREFERRED_WG_SIZE 128u @@ -1189,9 +1190,10 @@ struct ggml_webgpu_soft_max_pipeline_key_hash { inline bool ggml_webgpu_can_use_mmvq(const ggml_tensor * src0, const ggml_tensor * src1, + int64_t n_rows, bool supports_dot_product, const std::string & vendor) { - if (src1->ne[1] <= 4) { + if (n_rows <= GGML_WEBGPU_MUL_MAT_VEC_MAX_ROWS) { bool supports_dp4a = vendor == "amd" || vendor == "intel" || vendor == "nvidia"; if (supports_dp4a && supports_dot_product) { switch (src1->type) { @@ -2011,13 +2013,13 @@ class ggml_webgpu_shader_lib { ggml_webgpu_mul_mat_vec_pipeline_key key = {}; key.src0_type = context.src0->type; key.src1_type = context.src1->type; - key.vectorized = (context.src0->ne[0] % 4 == 0 && - (context.src0->type == GGML_TYPE_F32 || context.src0->type == GGML_TYPE_F16)) ? - 1 : - 0; - key.num_cols = context.dst->ne[1]; - key.use_mmvq = - ggml_webgpu_can_use_mmvq(context.src0, context.src1, context.supports_dot_product, context.vendor); + key.vectorized = (context.src0->ne[0] % 4 == 0 && + (context.src0->type == GGML_TYPE_F32 || context.src0->type == GGML_TYPE_F16)) ? + 1 : + 0; + key.num_cols = context.dst->ne[1]; + key.use_mmvq = ggml_webgpu_can_use_mmvq(context.src0, context.src1, context.src1->ne[1], + context.supports_dot_product, context.vendor); key.src_overlap = src_overlap; auto it = mul_mat_vec_pipelines.find(key); diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index 53cf86f81958..fac6cb60b0c6 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -1634,11 +1634,11 @@ static webgpu_encoded_op ggml_webgpu_mul_mat(webgpu_context & ctx, ggml_tensor * src1, ggml_tensor * dst) { // Determine if this is a mat-vec operation - bool use_mat_vec = (dst->ne[1] <= 4); + bool use_mat_vec = (dst->ne[1] <= GGML_WEBGPU_MUL_MAT_VEC_MAX_ROWS); // use MMVQ path for mat-vec - bool use_mmvq = ggml_webgpu_can_use_mmvq(src0, src1, ctx->global_ctx->capabilities.supports_dot_product, - ctx->global_ctx->vendor); + bool use_mmvq = ggml_webgpu_can_use_mmvq( + src0, src1, src1->ne[1], ctx->global_ctx->capabilities.supports_dot_product, ctx->global_ctx->vendor); ggml_webgpu_shader_lib_context shader_lib_ctx = {}; @@ -1965,7 +1965,8 @@ struct ggml_webgpu_flash_attn_op { static bool ggml_webgpu_flash_attn_use_vec_path(const webgpu_global_context & global_ctx, const ggml_tensor * Q, const ggml_tensor * K, - const ggml_tensor * V) { + const ggml_tensor * V, + int64_t n_q) { const size_t storage_offset_alignment = global_ctx->capabilities.limits.minStorageBufferOffsetAlignment; const bool k_float_vec4_aligned = (K->type != GGML_TYPE_F16 && K->type != GGML_TYPE_F32) || @@ -1979,7 +1980,7 @@ static bool ggml_webgpu_flash_attn_use_vec_path(const webgpu_global_context & gl ggml_is_quantized(V->type) ? ggml_blck_size(V->type) : GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH; const bool kv_vec_head_dims_aligned = Q->ne[0] % k_vec_head_align == 0 && V->ne[0] % v_vec_head_align == 0; - return global_ctx->capabilities.supports_subgroups && (Q->ne[1] < GGML_WEBGPU_FLASH_ATTN_VEC_MAX_SEQ_LEN) && + return global_ctx->capabilities.supports_subgroups && (n_q < GGML_WEBGPU_FLASH_ATTN_VEC_MAX_SEQ_LEN) && kv_vec_head_dims_aligned && k_float_vec4_aligned && v_float_vec4_aligned; } @@ -2288,7 +2289,7 @@ static webgpu_encoded_op ggml_webgpu_flash_attn(webgpu_context & ctx, ggml_tensor * sinks, ggml_tensor * dst) { ggml_webgpu_flash_attn_op op = ggml_webgpu_flash_attn_prepare(ctx, Q, K, V, mask, sinks, dst); - if (ggml_webgpu_flash_attn_use_vec_path(ctx->global_ctx, Q, K, V)) { + if (ggml_webgpu_flash_attn_use_vec_path(ctx->global_ctx, Q, K, V, Q->ne[1])) { return ggml_webgpu_flash_attn_vec(ctx, Q, K, V, mask, sinks, dst, std::move(op)); } return ggml_webgpu_flash_attn_direct(ctx, op); @@ -4012,7 +4013,10 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer const ggml_tensor * V = tensor->src[2]; const ggml_tensor * mask = tensor->src[3]; const auto & capabilities = ctx->webgpu_global_ctx->capabilities; - if (ggml_webgpu_flash_attn_use_vec_path(ctx->webgpu_global_ctx, Q, K, V)) { + // the vec path scratch grows with the query rows, so a longer batch reserves the scratch of the + // longest vec batch and the size stays monotonic in the batch length + const int64_t n_q = std::min(Q->ne[1], GGML_WEBGPU_FLASH_ATTN_VEC_MAX_SEQ_LEN - 1); + if (ggml_webgpu_flash_attn_use_vec_path(ctx->webgpu_global_ctx, Q, K, V, n_q)) { const bool kv_direct = ggml_webgpu_flash_attn_k_direct(Q, K, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH) || ggml_webgpu_flash_attn_v_direct(Q, V, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH); @@ -4024,7 +4028,7 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer uint32_t nwg = ggml_webgpu_flash_attn_vec_nwg(vec_nwg_cap, kv_tile, (uint32_t) K->ne[1]); const size_t align = capabilities.limits.minStorageBufferOffsetAlignment; - const uint64_t nrows = (uint64_t) Q->ne[1] * Q->ne[2] * Q->ne[3]; + const uint64_t nrows = (uint64_t) n_q * Q->ne[2] * Q->ne[3]; if (nwg > 1u) { const uint64_t tmp_data_elems = nrows * (uint64_t) V->ne[0] * nwg; const uint64_t tmp_stats_elems = nrows * 2u * nwg; @@ -4036,7 +4040,7 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer } if (mask != nullptr) { const uint32_t blk_nblk0 = CEIL_DIV((uint32_t) K->ne[1], kv_tile); - const uint32_t blk_nblk1 = CEIL_DIV((uint32_t) Q->ne[1], 1u); + const uint32_t blk_nblk1 = (uint32_t) n_q; const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type)); const uint32_t blk_batch_count = stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u; const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count; @@ -4052,11 +4056,15 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer { const ggml_tensor * src0 = tensor->src[0]; const ggml_tensor * src1 = tensor->src[1]; - bool use_mmvq = - ggml_webgpu_can_use_mmvq(src0, src1, ctx->webgpu_global_ctx->capabilities.supports_dot_product, - ctx->webgpu_global_ctx->vendor); + + // the q8 copy of src1 grows with its rows, so a longer batch reserves the copy of the longest + // mmvq batch and the size stays monotonic in the batch length + const int64_t n_rows = std::min(src1->ne[1], GGML_WEBGPU_MUL_MAT_VEC_MAX_ROWS); + const bool use_mmvq = ggml_webgpu_can_use_mmvq( + src0, src1, n_rows, ctx->webgpu_global_ctx->capabilities.supports_dot_product, + ctx->webgpu_global_ctx->vendor); if (use_mmvq) { - const size_t q8_src1_size = src1->ne[3] * src1->ne[2] * src1->ne[1] * + const size_t q8_src1_size = src1->ne[3] * src1->ne[2] * n_rows * (36 /* sizeof(q8_1) */ * (src1->ne[0] / /* block_size */ 32)); res = ROUNDUP_POW2(res + q8_src1_size + ctx->webgpu_global_ctx->capabilities.limits.minStorageBufferOffsetAlignment, From 4e910a86e8e8818e3619738ae372563a4e0cb42b Mon Sep 17 00:00:00 2001 From: o7si <32285332+o7si@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:34:17 +0800 Subject: [PATCH 15/36] cmake : add explicit thread linkage (#31414) --- tests/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6a681afc6bbe..84133935600a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,3 +1,5 @@ +find_package(Threads REQUIRED) + llama_add_compile_flags() # Optional args: @@ -351,7 +353,7 @@ if (GGML_METAL AND NOT GGML_BACKEND_DL) target_include_directories(test-metal-graph-optimize PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src ${PROJECT_SOURCE_DIR}/ggml/src/ggml-metal) endif() -llama_build(test-backend-ops.cpp LINK ggml) +llama_build(test-backend-ops.cpp LINK ggml Threads::Threads) llama_build_and_test(test-backend-tensor.cpp LINK ggml) From fcfe4b59df5374c6990c23ee56f9c8300b69968f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Mon, 2 Nov 2026 11:34:45 +0100 Subject: [PATCH 16/36] requirements : minor adjustments for python 3.14 wheels (#31534) --- requirements/requirements-tool_bench.txt | 3 +-- tools/server/tests/requirements.txt | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/requirements/requirements-tool_bench.txt b/requirements/requirements-tool_bench.txt index e92186a6de61..471f13fd55c1 100644 --- a/requirements/requirements-tool_bench.txt +++ b/requirements/requirements-tool_bench.txt @@ -1,10 +1,9 @@ -aiohttp~=3.9.3 pytest~=8.3.3 matplotlib~=3.10.0 numpy~=2.2.6; python_version < "3.14" numpy~=2.4.6; python_version >= "3.14" openai~=2.14.0 -pandas~=2.2.3 +pandas~=2.3.3 prometheus-client~=0.20.0 requests~=2.32.3 wget~=3.2 diff --git a/tools/server/tests/requirements.txt b/tools/server/tests/requirements.txt index e437876cabdd..3cea52523740 100644 --- a/tools/server/tests/requirements.txt +++ b/tools/server/tests/requirements.txt @@ -1,4 +1,3 @@ -aiohttp~=3.9.3 pytest~=8.3.3 pytest-xdist~=3.6 filelock~=3.16 From f72dc30928f27e7578751892a4ca42219e3a4ffb Mon Sep 17 00:00:00 2001 From: Li Chuyang <1418587903@qq.com> Date: Mon, 2 Nov 2026 18:36:45 +0800 Subject: [PATCH 17/36] docs: fix Huawei ascend brand (#31463) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ff7221db733d..d3199305d24f 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or | --- | --- | | [BLAS](docs/build.md#blas-build) | All | | [BLIS](docs/backend/BLIS.md) | All | -| [CANN](docs/build.md#cann) | Ascend NPU | +| [CANN](docs/build.md#cann) | Huawei Ascend NPU | | [CUDA](docs/build.md#cuda) | Nvidia GPU | | [HIP](docs/build.md#hip) | AMD GPU | | [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon | From a7a490a4905c36349f1074d3f5322291304a6dd3 Mon Sep 17 00:00:00 2001 From: Uber <20715993+YagUber@users.noreply.github.com> Date: Mon, 2 Nov 2026 04:39:53 -0600 Subject: [PATCH 18/36] ggml : fix clang-cl build with AVX512 BF16 (#31276) * Correctly gate immintrin.h inclusion when building with clang-cl and avx512bf16 * ggml : explicit vector casts for clang-cl --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 2 +- ggml/src/ggml.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 0d48c9a22e3d..865847a7785f 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -30,7 +30,7 @@ struct ggml_compute_params { }; -#if defined(_MSC_VER) +#if defined(_MSC_VER) && !defined(__clang__) #define m512bh(p) p #define m512i(p) p diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index e00f8e23afbf..822c8ab64982 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -71,12 +71,12 @@ uint64_t ggml_graph_next_uid(void) { // Needed for ggml_fp32_to_bf16_row() #if defined(__AVX512BF16__) -#if defined(_MSC_VER) +#if defined(_MSC_VER) && !defined(__clang__) #define m512i(p) p #else #include #define m512i(p) (__m512i)(p) -#endif // defined(_MSC_VER) +#endif // defined(_MSC_VER) && !defined(__clang__) #endif // defined(__AVX512BF16__) #if defined(__linux__) || \ From 59695669f1924c81f421f2044c690ed6373bec79 Mon Sep 17 00:00:00 2001 From: MuoDoo <36472129+MuoDoo@users.noreply.github.com> Date: Mon, 2 Nov 2026 18:45:40 +0800 Subject: [PATCH 19/36] jinja : implement map('filter') (#24033) * jinja: implement map('filter') * jinja : validate filter keyword arguments Assisted-by: Codex --- common/jinja/runtime.cpp | 50 +--------------- common/jinja/value.cpp | 124 +++++++++++++++++++++++++++++++++++---- common/jinja/value.h | 4 ++ tests/test-jinja.cpp | 114 +++++++++++++++++++++++++++++++++++ 4 files changed, 234 insertions(+), 58 deletions(-) diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index 7ec6606d7105..d32c79fa5cb5 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -307,58 +307,13 @@ value binary_expression::execute_impl(context & ctx) const { throw std::runtime_error("Unknown operator \"" + op.value + "\" between " + left_val->type() + " and " + right_val->type()); } -static value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false) { - JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str()); - if (ctx.is_get_stats) { - value_t::stats_t::mark_used(input); - input->stats.ops.insert(name); - } - auto builtins = input->get_builtins(); - auto it = builtins.find(name); - if (it != builtins.end()) { - JJ_DEBUG("Binding built-in '%s'", name.c_str()); - return mk_val(name, it->second, input); - } - if (undef_on_missing) { - return mk_val(name); - } - throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type()); -} - static value apply_filter(context & ctx, const statement_ptr & filter, value input) { JJ_DEBUG("Applying filter to %s", input->type().c_str()); - auto set_filter_alias = [](auto & filter_id) { - if (filter_id == "count") { - filter_id = "length"; - } else if (filter_id == "d") { - filter_id = "default"; - } else if (filter_id == "e") { - filter_id = "escape"; - } else if (filter_id == "trim") { - filter_id = "strip"; - } - }; - if (is_stmt(filter)) { auto filter_id = cast_stmt(filter)->val; - - set_filter_alias(filter_id); JJ_DEBUG("Applying filter '%s' to %s", filter_id.c_str(), input->type().c_str()); - // TODO: Refactor filters so this coercion can be done automatically - if (!input->is_undefined() && !is_val(input) && ( - filter_id == "capitalize" || - filter_id == "lower" || - filter_id == "replace" || - filter_id == "strip" || - filter_id == "title" || - filter_id == "upper" || - filter_id == "wordcount" - )) { - JJ_DEBUG("Coercing %s to String for '%s' filter", input->type().c_str(), filter_id.c_str()); - input = mk_val(input->as_string()); - } - return try_builtin_func(ctx, filter_id, input)->invoke(func_args(ctx)); + return apply_filter_by_name(ctx, filter_id, input, func_args(ctx)); } else if (is_stmt(filter)) { auto call = cast_stmt(filter); @@ -367,14 +322,13 @@ static value apply_filter(context & ctx, const statement_ptr & filter, value inp } auto filter_id = cast_stmt(call->callee)->val; - set_filter_alias(filter_id); JJ_DEBUG("Applying filter '%s' with arguments to %s", filter_id.c_str(), input->type().c_str()); func_args args(ctx); for (const auto & arg_expr : call->args) { args.push_back(arg_expr->execute(ctx)); } - return try_builtin_func(ctx, filter_id, input)->invoke(args); + return apply_filter_by_name(ctx, filter_id, input, args); } else { throw std::runtime_error("Invalid filter expression"); diff --git a/common/jinja/value.cpp b/common/jinja/value.cpp index 3aceca1e0177..333241d7a9e7 100644 --- a/common/jinja/value.cpp +++ b/common/jinja/value.cpp @@ -72,6 +72,17 @@ const std::vector & func_args::get_args() const { return args; } +void func_args::ensure_kwargs(std::initializer_list names) const { + for (const auto & arg : args) { + if (is_val(arg)) { + auto * kwarg = cast_val(arg); + if (std::find(names.begin(), names.end(), kwarg->key) == names.end()) { + throw raised_exception("Function '" + func_name + "' got an unexpected keyword argument '" + kwarg->key + "'"); + } + } + } +} + /** * Function that mimics Python's array slicing. */ @@ -134,6 +145,14 @@ static value empty_value_fn(const func_args &) { return mk_val(); } } + +template +static value empty_value_no_args_fn(const func_args & args) { + args.ensure_kwargs(); + args.ensure_count(1, 1); + return empty_value_fn(args); +} + template static value test_type_fn(const func_args & args) { args.ensure_count(1); @@ -594,6 +613,52 @@ const func_builtins & global_builtins() { return builtins; } +value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing) { + JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str()); + if (ctx.is_get_stats) { + value_t::stats_t::mark_used(input); + input->stats.ops.insert(name); + } + auto builtins = input->get_builtins(); + auto it = builtins.find(name); + if (it != builtins.end()) { + JJ_DEBUG("Binding built-in '%s'", name.c_str()); + return mk_val(name, it->second, input); + } + if (undef_on_missing) { + return mk_val(name); + } + throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type()); +} + +value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args) { + if (filter_id == "count") { + filter_id = "length"; + } else if (filter_id == "d") { + filter_id = "default"; + } else if (filter_id == "e") { + filter_id = "escape"; + } else if (filter_id == "trim") { + filter_id = "strip"; + } + + // TODO: Refactor filters so this coercion can be done automatically + if (!input->is_undefined() && !is_val(input) && ( + filter_id == "capitalize" || + filter_id == "lower" || + filter_id == "replace" || + filter_id == "strip" || + filter_id == "title" || + filter_id == "upper" || + filter_id == "wordcount" + )) { + JJ_DEBUG("Coercing %s to String for '%s' filter", input->type().c_str(), filter_id.c_str()); + input = mk_val(input->as_string()); + } + + return try_builtin_func(ctx, filter_id, input)->invoke(args); +} + const func_builtins & value_int_t::get_builtins() const { static const func_builtins builtins = { @@ -667,16 +732,22 @@ const func_builtins & value_string_t::get_builtins() const { static const func_builtins builtins = { {"default", default_value}, {"upper", [](const func_args & args) -> value { + args.ensure_kwargs(); + args.ensure_count(1, 1); args.ensure_vals(); jinja::string str = args.get_pos(0)->as_string().uppercase(); return mk_val(str); }}, {"lower", [](const func_args & args) -> value { + args.ensure_kwargs(); + args.ensure_count(1, 1); args.ensure_vals(); jinja::string str = args.get_pos(0)->as_string().lowercase(); return mk_val(str); }}, {"strip", [](const func_args & args) -> value { + args.ensure_kwargs({"chars"}); + args.ensure_count(1, 2); value val_input = args.get_pos(0); if (!is_val(val_input)) { throw raised_exception("strip() first argument must be a string"); @@ -707,11 +778,15 @@ const func_builtins & value_string_t::get_builtins() const { } }}, {"title", [](const func_args & args) -> value { + args.ensure_kwargs(); + args.ensure_count(1, 1); args.ensure_vals(); jinja::string str = args.get_pos(0)->as_string().titlecase(); return mk_val(str); }}, {"capitalize", [](const func_args & args) -> value { + args.ensure_kwargs(); + args.ensure_count(1, 1); args.ensure_vals(); jinja::string str = args.get_pos(0)->as_string().capitalize(); return mk_val(str); @@ -1013,6 +1088,10 @@ const func_builtins & value_bool_t::get_builtins() const { return builtins; } +[[noreturn]] static value array_sum_not_implemented(const func_args &) { + throw not_implemented_exception("Array sum builtin not implemented"); +} + const func_builtins & value_array_t::get_builtins() const { static const func_builtins builtins = { {"default", default_value}, @@ -1079,7 +1158,9 @@ const func_builtins & value_array_t::get_builtins() const { {"select", select_or_reject}, {"rejectattr", select_or_reject}, {"reject", select_or_reject}, + {"sum", array_sum_not_implemented}, {"join", [](const func_args & args) -> value { + args.ensure_kwargs({"d", "attribute"}); args.ensure_count(1, 3); if (!is_val(args.get_pos(0))) { throw raised_exception("join() first argument must be an array"); @@ -1117,13 +1198,28 @@ const func_builtins & value_array_t::get_builtins() const { {"tojson", tojson}, {"map", [](const func_args & args) -> value { args.ensure_count(2); - if (!is_val(args.get_pos(0))) { + value val = args.get_pos(0); + if (!is_val(val)) { throw raised_exception("map: first argument must be an array"); } if (!is_val(args.get_args().at(1))) { - throw not_implemented_exception("map: filter-mapping not implemented"); + value filter = args.get_pos(1); + if (!is_val(filter)) { + throw raised_exception("map: filter name must be a string"); + } + auto out = mk_val(); + func_args filter_args(args.ctx); + const auto & all_args = args.get_args(); + for (size_t i = 2; i < all_args.size(); ++i) { + filter_args.push_back(all_args[i]); + } + const std::string filter_id = filter->as_string().str(); + auto arr = val->as_array(); + for (const auto & item : arr) { + out->push_back(apply_filter_by_name(args.ctx, filter_id, item, filter_args)); + } + return is_val(val) ? mk_val(out->as_array()) : out; } - value val = args.get_pos(0); value attribute = args.get_kwarg_or_pos("attribute", 1); value default_val = args.get_kwarg("default", mk_val()); auto out = mk_val(); @@ -1132,7 +1228,7 @@ const func_builtins & value_array_t::get_builtins() const { value attr_val = get_attribute(item, attribute, default_val); out->push_back(attr_val); } - return is_val(val) ? mk_val(std::move(out->as_array())) : out; + return is_val(val) ? mk_val(out->as_array()) : out; }}, {"append", [](const func_args & args) -> value { args.ensure_count(2); @@ -1405,14 +1501,18 @@ const func_builtins & value_none_t::get_builtins() const { const func_builtins & value_undefined_t::get_builtins() const { static const func_builtins builtins = { {"default", default_value}, - {"capitalize", empty_value_fn}, + {"capitalize", empty_value_no_args_fn}, {"first", empty_value_fn}, {"items", empty_value_fn}, - {"join", empty_value_fn}, + {"join", [](const func_args & args) -> value { + args.ensure_kwargs({"d", "attribute"}); + args.ensure_count(1, 3); + return empty_value_fn(args); + }}, {"last", empty_value_fn}, {"length", empty_value_fn}, {"list", empty_value_fn}, - {"lower", empty_value_fn}, + {"lower", empty_value_no_args_fn}, {"map", empty_value_fn}, {"max", empty_value_fn}, {"min", empty_value_fn}, @@ -1425,12 +1525,16 @@ const func_builtins & value_undefined_t::get_builtins() const { {"selectattr", empty_value_fn}, {"sort", empty_value_fn}, {"string", empty_value_fn}, - {"strip", empty_value_fn}, + {"strip", [](const func_args & args) -> value { + args.ensure_kwargs({"chars"}); + args.ensure_count(1, 2); + return empty_value_fn(args); + }}, {"sum", empty_value_fn}, - {"title", empty_value_fn}, + {"title", empty_value_no_args_fn}, {"truncate", empty_value_fn}, {"unique", empty_value_fn}, - {"upper", empty_value_fn}, + {"upper", empty_value_no_args_fn}, {"wordcount", empty_value_fn}, }; return builtins; diff --git a/common/jinja/value.h b/common/jinja/value.h index 9bb1a87bd018..8fdff884edee 100644 --- a/common/jinja/value.h +++ b/common/jinja/value.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -176,6 +177,8 @@ struct value_t { // const func_builtins & global_builtins(); +value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false); +value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args); std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": "); @@ -658,6 +661,7 @@ struct func_args { size_t count() const { return args.size(); } void push_back(const value & val); void push_front(const value & val); + void ensure_kwargs(std::initializer_list names = {}) const; void ensure_count(size_t min, size_t max = 999) const { size_t n = args.size(); if (n < min || n > max) { diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp index 444d57fbe397..ce5a9749ba94 100644 --- a/tests/test-jinja.cpp +++ b/tests/test-jinja.cpp @@ -677,6 +677,24 @@ static void test_set_statement(testing & t) { } static void test_filters(testing & t) { + for (const std::string filter : {"upper", "lower", "title", "capitalize"}) { + for (const std::string input : {"'hello'", "missing"}) { + test_template_error(t, filter + " rejects unknown keyword on " + input, + "{{ " + input + "|" + filter + "(attribute='name') }}", + "unexpected keyword argument 'attribute'" + ); + test_template_error(t, filter + " rejects extra positional argument on " + input, + "{{ " + input + "|" + filter + "('extra') }}", + "expected between 1 and 1 arguments, got 2" + ); + } + } + + test_template_error(t, "string method rejects unknown keyword", + "{{ 'hello'.upper(attribute='name') }}", + "unexpected keyword argument 'attribute'" + ); + test_template(t, "upper", "{{ 'hello'|upper }}", json::object(), @@ -725,6 +743,23 @@ static void test_filters(testing & t) { "hello" ); + test_template(t, "trim chars keyword", + "{{ 'xyxhelloxyx'|trim(chars='xy') }}|{{ missing|trim(chars='xy') }}", + json::object(), + "hello|" + ); + + for (const std::string input : {"'hello'", "missing"}) { + test_template_error(t, "trim rejects unknown keyword on " + input, + "{{ " + input + "|trim(attribute='name') }}", + "unexpected keyword argument 'attribute'" + ); + test_template_error(t, "trim rejects extra positional argument on " + input, + "{{ " + input + "|trim('x', 'y') }}", + "expected between 1 and 2 arguments, got 3" + ); + } + test_template(t, "length string", "{{ 'hello'|length }}", json::object(), @@ -2064,6 +2099,79 @@ static void test_array_methods(testing & t) { "{\"x\": 2}" ); + test_template(t, "array|map with filter int materialized", + "{{ arr|map('int')|list|join(',') }}", + {{"arr", json::array({"1", "2", "3"})}}, + "1,2,3" + ); + + test_template(t, "array|map with filter upper for schema union type", + "{{ item_value|map('upper')|list|join(',') }}", + {{"item_value", json::array({"string", "null"})}}, + "STRING,NULL" + ); + + test_template_error(t, "array|map rejects unsupported filter attribute", + "{{ [{'name': 'alice'}]|map('upper', attribute='name')|join(',') }}", + "unexpected keyword argument 'attribute'" + ); + + test_template_error(t, "array|map rejects unknown keyword on undefined item", + "{{ [missing]|map('lower', attribute='name')|join(',') }}", + "unexpected keyword argument 'attribute'" + ); + + test_template(t, "array|map attribute then filter upper", + "{{ arr|map(attribute='name')|map('upper')|join(',') }}", + {{"arr", json::array({{{"name", "alice"}}, {{"name", "bob"}}})}}, + "ALICE,BOB" + ); + + test_template(t, "array|map with filter arguments", + "{{ arr|map('replace', 'a', 'o')|join(',') }}", + {{"arr", json::array({"cat", "bat"})}}, + "cot,bot" + ); + + test_template(t, "array|map with filter keyword arguments", + "{{ arr|map('join', '-', attribute='name')|join(';') }}", + {{"arr", json::array({ + json::array({ + json({{"name", "a"}}), + json({{"name", "b"}}), + }), + json::array({ + json({{"name", "c"}}), + }), + })}}, + "a-b;c" + ); + + test_template(t, "array|map with count alias", + "{{ arr|map('count')|join(',') }}", + {{"arr", json::array({"cat", "hello"})}}, + "3,5" + ); + + test_template(t, "array|map with join keyword arguments", + "{{ [[{'name': 'a'}, {'name': 'b'}]]|map('join', d='-', attribute='name')|join(';') }}", + json::object(), + "a-b" + ); + + for (const std::string input : {"['a', 'b']", "[]", "missing"}) { + test_template_error(t, "join rejects unknown keyword on " + input, + "{{ " + input + "|join(typo='name') }}", + "unexpected keyword argument 'typo'" + ); + } + + test_template(t, "array|map with default alias and arguments", + "{{ arr|map('d', 'fallback', true)|join(',') }}", + {{"arr", json::array({"", "value"})}}, + "fallback,value" + ); + // not used by any chat templates // test_template(t, "array.insert()", // "{% set _ = arr.insert(1, 'x') %}{{ arr|join(',') }}", @@ -2131,6 +2239,12 @@ static void test_array_methods(testing & t) { "" ); + test_template(t, "undefined|join with keyword arguments", + "{{ arr|join(d='-', attribute='name') }}", + json::object(), + "" + ); + test_template(t, "undefined|sort", "{{ arr|sort|string }}", json::object(), From 00ef136649bd9955bfe11fb9c5a6028aee1a1185 Mon Sep 17 00:00:00 2001 From: Niklas Wenzel Date: Mon, 2 Nov 2026 11:59:00 +0100 Subject: [PATCH 20/36] tests : merge identical if branches (#31523) --- tests/test-llama-archs.cpp | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 312733ecd22e..a7ae38132fa4 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -423,13 +423,6 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { if (arch == LLM_ARCH_MAION_CODER) { ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector{1, 1, 0, 0}); - } else if (arch == LLM_ARCH_BEAM) { - std::vector pattern; - pattern.reserve(n_layer); - for (uint32_t il = 0; il < n_layer; il++) { - pattern.push_back(il % 2); - } - ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern); } else if (arch == LLM_ARCH_GEMMA4) { ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, n_embd/2); ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(5)); @@ -445,7 +438,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern); } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE || - arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT) { + arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT || arch == LLM_ARCH_BEAM) { std::vector pattern; pattern.reserve(n_layer); for (uint32_t il = 0; il < n_layer; il++) { From aac6da8012efd5c0fd211c907c4e870ba075b211 Mon Sep 17 00:00:00 2001 From: Liu Jia Date: Mon, 2 Nov 2026 19:01:33 +0800 Subject: [PATCH 21/36] docs : fix Vulkan example command in build.md (#31530) --- docs/build.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build.md b/docs/build.md index 42c726127af6..4e3bfe651b39 100644 --- a/docs/build.md +++ b/docs/build.md @@ -481,7 +481,7 @@ cmake --build build --config Release Now you can load the model in conversation mode using `Vulkan` ```sh -build/bin/Release/llama-cli -m "[PATH TO MODEL]" -ngl 100 -c 16384 -t 10 -n -2 -cnv +build/bin/Release/llama-cli -m "[PATH TO MODEL]" -ngl 100 -c 16384 -t 10 -n -2 ``` **MSYS2** From b12b405c171bc2ec98ce7a860a69c073dcac41b3 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 2 Nov 2026 12:33:28 +0100 Subject: [PATCH 22/36] ui: don't reload a loading model on select, prefer it in pre-selection (#31326) * ui: don't reload a loading model on select, prefer it in pre-selection Assisted-by: Claude (Opus 5.5 Medium) * ui: skip hidden models in automatic model selection Assisted-by: Claude (Opus 5.5 Medium) * ui: don't count a model being unloaded as loading Assisted-by: Claude (Opus 5.5 Medium) --- .../lib/hooks/use-chat-form-model.svelte.ts | 16 ++-- .../lib/hooks/use-models-selector.svelte.ts | 10 ++- .../ui/src/lib/stores/models/index.svelte.ts | 82 ++++++++++++++++--- 3 files changed, 87 insertions(+), 21 deletions(-) diff --git a/tools/ui/src/lib/hooks/use-chat-form-model.svelte.ts b/tools/ui/src/lib/hooks/use-chat-form-model.svelte.ts index a32acba6aa5b..c86f492a8df8 100644 --- a/tools/ui/src/lib/hooks/use-chat-form-model.svelte.ts +++ b/tools/ui/src/lib/hooks/use-chat-form-model.svelte.ts @@ -2,10 +2,10 @@ * Model state for the chat form's action bar. * * Syncs the conversation's model into the global selection (a conversation - * opened from history names the model that produced it), falls back to the - * first loaded model on a router, and reactively tracks which modalities - * (vision / audio / video) the active model supports - fetching model props - * from the server on demand if they are not cached yet. + * opened from history names the model that produced it), falls back to a + * loading, then the first loaded, model on a router, and reactively tracks + * which modalities (vision / audio / video) the active model supports - + * fetching model props from the server on demand if they are not cached yet. */ import { conversationsStore, modelsStore, serverStore } from '$lib/stores'; @@ -36,12 +36,16 @@ export function useChatFormModel() { } else if ( isRouter && !modelsStore.selectedModelId && - modelsStore.loadedModelIds.length > 0 && conversationsStore.activeMessages.length > 0 && !conversationModel ) { lastSyncedConversationModel = null; - const first = modelsStore.models.find((m) => modelsStore.loadedModelIds.includes(m.model)); + // a model that is loading wins over one that is already up; a model the + // selector doesn't list is never picked + const visible = modelsStore.getVisibleModels(); + const first = + visible.find((m) => modelsStore.isModelLoading(m.model)) ?? + visible.find((m) => modelsStore.loadedModelIds.includes(m.model)); if (first) modelsStore.selectModelById(first.id); } diff --git a/tools/ui/src/lib/hooks/use-models-selector.svelte.ts b/tools/ui/src/lib/hooks/use-models-selector.svelte.ts index 844a7d4fbfb8..6fe35e444319 100644 --- a/tools/ui/src/lib/hooks/use-models-selector.svelte.ts +++ b/tools/ui/src/lib/hooks/use-models-selector.svelte.ts @@ -167,8 +167,14 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele }); } - // only the built-in server loads on request, and only in router mode - if (!onModelChange && isRouter && !modelsStore.isModelLoaded(option.model)) { + // only the built-in server loads on request, and only in router mode; a model + // that is already loading is only selected, not loaded a second time + if ( + !onModelChange && + isRouter && + !modelsStore.isModelLoaded(option.model) && + !modelsStore.isModelLoading(option.model) + ) { isLoadingModel = true; modelsStore.status diff --git a/tools/ui/src/lib/stores/models/index.svelte.ts b/tools/ui/src/lib/stores/models/index.svelte.ts index 79bd5b1ad2cc..0a683c24bc1b 100644 --- a/tools/ui/src/lib/stores/models/index.svelte.ts +++ b/tools/ui/src/lib/stores/models/index.svelte.ts @@ -205,12 +205,22 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { * Prioritizes: * 1. Model from active conversation's last assistant response (if loaded) * 2. Model from active conversation's last assistant response (if not loaded) - * 3. First loaded model (not from active conversation) - * 4. A favorite model - * 5. First available model + * 3. A model that is loading + * 4. First loaded model (not from active conversation) + * 5. A favorite model + * 6. First available model + * + * A chat without a model of its own also trades an idle selection for a + * model that is loading. */ async ensureFirstModelSelected(): Promise { - if (this.selectedModelName) return; + if (this.selectedModelName) { + const loadingModel = this.getLoadingModelToPreselect(); + + if (loadingModel) await this.selectModelById(loadingModel.id); + + return; + } const availableModels = this.getVisibleModels(); @@ -233,6 +243,15 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { } } + // a model that is loading is the one the user is waiting for + const loadingModel = availableModels.find((m) => this.isModelLoading(m.model)); + + if (loadingModel) { + await this.selectModelById(loadingModel.id); + + return; + } + // Try a loaded model first const loadedModel = availableModels.find((m) => this.isModelLoaded(m.model)); @@ -321,6 +340,20 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { ); } + /** + * The loading model a chat without a model of its own should switch to: none + * while the current selection is loaded or loading itself. + */ + getLoadingModelToPreselect(): ModelOption | null { + if (this.getModelFromLastAssistantResponse()) return null; + + const selected = this.selectedModelName; + + if (selected && (this.isModelLoaded(selected) || this.isModelLoading(selected))) return null; + + return this.getVisibleModels().find((m) => this.isModelLoading(m.model)) ?? null; + } + /** * Gets the model name from the last assistant message in the active conversation. * Used by both the chat page and settings page to maintain model consistency. @@ -345,6 +378,16 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { return (model?.status?.value as ServerModelStatus) ?? null; } + /** + * Models the selector lists: not flagged out of the UI (ui === false) and not + * hidden by the user, so an automatic pick never lands on one it doesn't show. + */ + getVisibleModels(): ModelOption[] { + return this.models.filter( + (option) => this.props.getModelProps(option.model)?.ui !== false && !this.isHidden(option.id) + ); + } + hasModel(modelName: string): boolean { return this.models.some((model) => model.model === modelName); } @@ -367,6 +410,15 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { return status === ServerModelStatus.LOADED || status === ServerModelStatus.SLEEPING; } + /** Loading on the server, or a load this client started that has not finished yet. */ + isModelLoading(modelId: string): boolean { + // an unload raises the same in-progress flag, but the model stays loaded until it ends + return ( + this.getModelStatus(modelId) === ServerModelStatus.LOADING || + (this._status.isOperationInProgress(modelId) && !this.isModelLoaded(modelId)) + ); + } + /** Loaded, or sleeping, and not mid-operation: what a row shows as running. */ isModelRunning(modelId: string): boolean { return this.isModelLoaded(modelId) && !this._status.isOperationInProgress(modelId); @@ -422,13 +474,24 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { } /** - * Auto-selects the model from the last assistant response if available and loaded. + * Auto-selects the model from the last assistant response if available and loaded, + * or a loading model when the conversation has no model yet. * Returns true if a model was selected, false otherwise. */ async selectModelFromLastAssistantResponse(): Promise { const lastModel = this.getModelFromLastAssistantResponse(); - if (!lastModel || this.selectedModelName === lastModel) return false; + if (!lastModel) { + const loadingModel = this.getLoadingModelToPreselect(); + + if (!loadingModel) return false; + + await this.selectModelById(loadingModel.id); + + return true; + } + + if (this.selectedModelName === lastModel) return false; const matchingModel = this.models.find((option) => option.model === lastModel); @@ -612,19 +675,12 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { }) ); } - /** Fetch models in MODEL mode (single model, standard OpenAI-compatible). */ private async fetchModelModeInternal(): Promise { const response = await ModelsService.list(); return this.buildModelOptions(response); } - /** - * Filter to models visible in the UI (ui !== false). - */ - private getVisibleModels(): ModelOption[] { - return this.models.filter((option) => this.props.getModelProps(option.model)?.ui !== false); - } private loadFavoritesFromStorage(): Set { try { From 64be95b163fe3a80327fad1ab84dc162302cf235 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 2 Nov 2026 12:34:00 +0100 Subject: [PATCH 23/36] ui: show interrupted state for tool calls that never returned (#31314) * ui: show interrupted state for tool calls that never returned The chat-level streaming flag was applied to every assistant message, so a tool call left without a result in an earlier message kept spinning while a newer reply streamed. Scope streaming to the last assistant message and mark finished calls without a result as interrupted. Assisted-by: Claude (Opus 5.5 Medium) * ui: mark a shell command stopped before any output as interrupted exec_shell_command creates its result message before running, so stopping it mid-run leaves an empty result instead of none. Assisted-by: Claude (Opus 5.5 Medium) --- ...essageToolCallBlockExecShellCommand.svelte | 9 ++- .../ChatMessageToolCall/ToolCallBlock.svelte | 18 ++++++ .../ChatMessageAgenticContent.svelte | 15 ++--- ...xec-shell-command-streaming.svelte.test.ts | 62 +++++++++++++++++++ .../tool-call-in-progress.svelte.test.ts | 9 +++ 5 files changed, 105 insertions(+), 8 deletions(-) diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte index 665238a29d9c..a67de857c6a7 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte @@ -8,7 +8,7 @@ import { parseExecShellCommandMeta } from './parsers/exec-shell-command'; import ToolCallBlock from './ToolCallBlock.svelte'; - import { AlertTriangle, Check, Loader2, XCircle } from '@lucide/svelte'; + import { AlertTriangle, Check, CircleStop, Loader2, XCircle } from '@lucide/svelte'; import { CollapsibleTerminalBlock } from '$lib/components/app'; import { SETTINGS_KEYS, TOOL_RUNTIME_SCROLL_AT_BOTTOM_THRESHOLD_PX } from '$lib/constants'; import { AgenticSectionType, AttachmentType } from '$lib/enums'; @@ -229,6 +229,7 @@ {/snippet} Executing command... + {:else if ctx.isInterrupted} +
+ + + Command interrupted before it returned output +
{:else if execShellError}
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ToolCallBlock.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ToolCallBlock.svelte index 16dd6e108837..461ffe6f7663 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ToolCallBlock.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ToolCallBlock.svelte @@ -25,6 +25,7 @@ isPending: boolean; isStreamingCall: boolean; isCodeStreaming: boolean; + isInterrupted: boolean; } interface Props { @@ -50,6 +51,12 @@ * inline within the body. */ spinIconWhenActive?: boolean; + /** + * Treat an empty result as interrupted too. For tools that create + * their result message before running (`exec_shell_command`), so a + * stop mid-run leaves an empty result rather than none. + */ + emptyResultIsInterrupted?: boolean; /** * Wrapper component that renders the title row and the body * children. Defaults to CollapsibleContentBlock; @@ -65,6 +72,7 @@ let { children, + emptyResultIsInterrupted = false, extraLiveStreaming = false, isStreaming, meta, @@ -81,6 +89,13 @@ const isStreamingCall = $derived(section.type === AgenticSectionType.TOOL_CALL_STREAMING); const showSpinner = $derived(isPending || (isStreamingCall && isStreaming) || extraLiveStreaming); const isCodeStreaming = $derived(isStreaming && (isPending || isStreamingCall)); + // A finished call with no result was stopped before it returned (user + // abort, or the page went away mid-execution). + const isInterrupted = $derived( + section.type === AgenticSectionType.TOOL_CALL && + (section.toolResult === undefined || (emptyResultIsInterrupted && !section.toolResult)) && + !showSpinner + ); const toolUi: ToolUiEntry | null = $derived(getToolUi(section.toolName)); const toolIcon: Component = $derived( @@ -107,6 +122,8 @@ if (isStreamingCall && !isStreaming) return 'incomplete'; + if (isInterrupted) return 'interrupted'; + return undefined; } @@ -126,6 +143,7 @@ > {@render children(meta, { isCodeStreaming, + isInterrupted, isPending, isStreaming, isStreamingCall diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte index b6138dfc0879..c2a8188ec798 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte @@ -26,11 +26,16 @@ let { isLastAssistantMessage = false, - isStreaming = false, + isStreaming: isChatStreaming = false, message, toolMessages = [] }: Props = $props(); + // The chat-level streaming flag is shared by every message; only the last + // assistant message is actually live, so earlier ones must not keep their + // unresolved tool calls spinning while a newer reply streams. + const isStreaming = $derived(isChatStreaming && isLastAssistantMessage); + let expandedStates: Record = $state({}); const showThoughtInProgress = $derived(Boolean(settingsStore.config.showThoughtInProgress)); @@ -48,9 +53,7 @@ ); const pendingPermission = $derived( - isStreaming && isLastAssistantMessage - ? agenticStore.getPendingPermissionRequest(message.convId) - : null + isStreaming ? agenticStore.getPendingPermissionRequest(message.convId) : null ); // dismissal applies to the request object, so the next request ( new @@ -67,9 +70,7 @@ } const pendingContinue = $derived( - isStreaming && isLastAssistantMessage - ? agenticStore.getPendingContinueRequest(message.convId) - : false + isStreaming ? agenticStore.getPendingContinueRequest(message.convId) : false ); let continueDismissed = $state(false); diff --git a/tools/ui/tests/client/exec-shell-command-streaming.svelte.test.ts b/tools/ui/tests/client/exec-shell-command-streaming.svelte.test.ts index 87668c0e30b6..e948cc9228a0 100644 --- a/tools/ui/tests/client/exec-shell-command-streaming.svelte.test.ts +++ b/tools/ui/tests/client/exec-shell-command-streaming.svelte.test.ts @@ -81,6 +81,7 @@ describe('exec_shell_command streaming title', () => { it('streams the partial command into the title with a cursor', async () => { const { container } = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, isStreaming: true, message: messageWithArgs('{"command":"git sta'), toolMessages: [] @@ -98,6 +99,7 @@ describe('exec_shell_command streaming title', () => { it('drops the cursor once the args JSON is complete', async () => { const screen = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, isStreaming: true, message: messageWithArgs('{"command":"git sta'), toolMessages: [] @@ -108,6 +110,7 @@ describe('exec_shell_command streaming title', () => { expect(hasCursor(screen.container)).toBe(true); screen.rerender({ + isLastAssistantMessage: true, isStreaming: true, message: messageWithArgs('{"command":"git status"}'), toolMessages: [] @@ -120,6 +123,7 @@ describe('exec_shell_command streaming title', () => { it('switches the prompt from the conversation cwd to the persisted one', async () => { const screen = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, isStreaming: true, message: messageWithArgs('{"command":"git status"}'), toolMessages: [] @@ -131,6 +135,7 @@ describe('exec_shell_command streaming title', () => { // Result message lands: its x-tool-cwd takes over the prompt. screen.rerender({ + isLastAssistantMessage: true, isStreaming: true, message: messageWithArgs('{"command":"git status"}'), toolMessages: [toolResultMessage('', '/other/dir')] @@ -143,6 +148,7 @@ describe('exec_shell_command streaming title', () => { it('shows the executing label only after the command finishes streaming', async () => { const screen = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, isStreaming: true, message: messageWithArgs('{"command":"git sta'), toolMessages: [] @@ -154,6 +160,7 @@ describe('exec_shell_command streaming title', () => { expect(screen.container.textContent).not.toContain('Executing command...'); screen.rerender({ + isLastAssistantMessage: true, isStreaming: true, message: messageWithArgs('{"command":"git status"}'), toolMessages: [] @@ -192,6 +199,7 @@ describe('exec_shell_command streaming title', () => { conversationsStore.activeMessages = [messageWithArgs('')]; const screen = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, isStreaming: true, message: conversationsStore.activeMessages[0]!, toolMessages: [] @@ -214,3 +222,57 @@ describe('exec_shell_command streaming title', () => { } }); }); + +describe('exec_shell_command interrupted call', () => { + beforeEach(() => { + settingsStore.updateMultipleConfig({ + [SETTINGS_KEYS.ALWAYS_SHOW_TOOL_CALL_CONTENT]: true, + [SETTINGS_KEYS.SHOW_TOOL_CALL_IN_PROGRESS]: true + }); + }); + + it('marks a call without a result as interrupted once streaming stops', async () => { + const { container } = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, + isStreaming: false, + message: messageWithArgs('{"command":"git status"}'), + toolMessages: [] + }); + + await tick(); + + expect(triggerText(container)).toContain('interrupted'); + expect(container.textContent).toContain('Command interrupted'); + expect(container.textContent).not.toContain('Executing command...'); + }); + + it('marks a command stopped before any output as interrupted', async () => { + // exec_shell_command creates its result message before running, so a + // stop mid-run leaves an empty result rather than none. + const { container } = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, + isStreaming: false, + message: messageWithArgs('{"command":"sleep 40"}'), + toolMessages: [toolResultMessage('', CWD)] + }); + + await tick(); + + expect(triggerText(container)).toContain('interrupted'); + expect(container.textContent).toContain('Command interrupted'); + }); + + it('does not keep spinning in an earlier message while a newer reply streams', async () => { + const { container } = render(ChatMessageAgenticContent, { + isLastAssistantMessage: false, + isStreaming: true, + message: messageWithArgs('{"command":"git status"}'), + toolMessages: [] + }); + + await tick(); + + expect(triggerText(container)).toContain('interrupted'); + expect(container.textContent).not.toContain('Executing command...'); + }); +}); diff --git a/tools/ui/tests/client/tool-call-in-progress.svelte.test.ts b/tools/ui/tests/client/tool-call-in-progress.svelte.test.ts index e51fcb795dc5..ffb28e369a43 100644 --- a/tools/ui/tests/client/tool-call-in-progress.svelte.test.ts +++ b/tools/ui/tests/client/tool-call-in-progress.svelte.test.ts @@ -95,6 +95,7 @@ describe('Show tool call in progress', () => { setSettings({ alwaysShow: false, inProgress: true }); const { container } = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, isStreaming: true, message: assistantMessage(), toolMessages: [] @@ -107,6 +108,7 @@ describe('Show tool call in progress', () => { it('keeps the call collapsed while executing when the setting is off', async () => { const { container } = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, isStreaming: true, message: assistantMessage(), toolMessages: [] @@ -123,6 +125,7 @@ describe('Show tool call in progress', () => { setSettings({ alwaysShow: false, inProgress: true }); const screen = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, isStreaming: true, message: assistantMessage(), toolMessages: [] @@ -135,6 +138,7 @@ describe('Show tool call in progress', () => { // Same block, now finished: it must close on its own, no toggle given. screen.rerender({ + isLastAssistantMessage: true, isStreaming: true, message: assistantMessage(), toolMessages: [toolResultMessage()] @@ -150,6 +154,7 @@ describe('Show tool call in progress', () => { setSettings({ alwaysShow: false, inProgress: true }); const screen = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, isStreaming: true, message: assistantMessage(), toolMessages: [] @@ -161,6 +166,7 @@ describe('Show tool call in progress', () => { await collapseBlock(screen); screen.rerender({ + isLastAssistantMessage: true, isStreaming: true, message: assistantMessage(), toolMessages: [toolResultMessage()] @@ -175,6 +181,7 @@ describe('Show tool call in progress', () => { setSettings({ alwaysShow: true, inProgress: false }); const { container } = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, isStreaming: true, message: assistantMessage(), toolMessages: [toolResultMessage()] @@ -189,6 +196,7 @@ describe('Show tool call in progress', () => { setSettings({ alwaysShow: false, inProgress: true }); const screen = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, isStreaming: true, message: assistantMessage(), toolMessages: [] @@ -206,6 +214,7 @@ describe('Show tool call in progress', () => { it('lets a manual expand win over the collapsed default', async () => { const screen = render(ChatMessageAgenticContent, { + isLastAssistantMessage: true, isStreaming: true, message: assistantMessage(), toolMessages: [] From 90080ce094236542470311c484543d507564d549 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 2 Nov 2026 12:34:17 +0100 Subject: [PATCH 24/36] ui: keep queued steering messages in their own conversation (#31304) A message queued while a conversation was busy was re-sent through sendMessage, which targets the active conversation, so switching chats before the turn ended delivered it into the wrong one. Re-send only when the owning conversation is active; otherwise keep it queued there and flush it when that conversation is opened again. Assisted-by: Claude (Opus 5.5 Medium) --- tools/ui/src/lib/stores/chat/index.svelte.ts | 40 ++++++++- .../src/routes/(chat)/chat/[id]/+page.svelte | 2 + .../unit/queued-message-isolation.test.ts | 81 +++++++++++++++++++ 3 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 tools/ui/tests/unit/queued-message-isolation.test.ts diff --git a/tools/ui/src/lib/stores/chat/index.svelte.ts b/tools/ui/src/lib/stores/chat/index.svelte.ts index d1a754aeaca0..ec20ecd34b06 100644 --- a/tools/ui/src/lib/stores/chat/index.svelte.ts +++ b/tools/ui/src/lib/stores/chat/index.svelte.ts @@ -337,7 +337,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { const pending = this.consumePendingMessage(convId); - if (pending) await this.sendMessage(pending.content, pending.extras); + if (pending) await this.sendQueuedMessage(convId, pending); } consumePendingMessage( @@ -416,6 +416,20 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { return this.flows.editUserMessagePreserveResponses(messageId, newContent, newExtras); } + /** + * Send the message left queued for `convId` once that conversation is + * active and idle (e.g. after navigating back to it). + */ + async flushPendingMessage(convId: string): Promise { + if (conversationsStore.activeConversation?.id !== convId) return; + + if (this.isChatLoadingInternal(convId) || agenticStore.isRunning(convId)) return; + + const pending = this.consumePendingMessage(convId); + + if (pending) await this.sendMessage(pending.content, pending.extras); + } + getAddFilesHandler(): ((files: File[]) => void) | null { return this.addFilesHandler; } @@ -1141,7 +1155,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { const pending = this.consumePendingMessage(convId); if (pending) { - this.sendMessage(pending.content, pending.extras); + void this.sendQueuedMessage(convId, pending); } return; @@ -1268,7 +1282,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { const pending = agenticStore.consumePendingSteeringMessage(convId); if (pending) { - await this.sendMessage(pending.content, pending.extras); + await this.sendQueuedMessage(convId, pending); } return; @@ -1333,7 +1347,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { const pending = this.consumePendingMessage(convId); if (pending) { - await this.sendMessage(pending.content, pending.extras); + await this.sendQueuedMessage(convId, pending); } }, onCompletionId: streamCallbacks.onCompletionId, @@ -1555,6 +1569,24 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { } } + /** + * Re-send a message queued while `convId` was busy. sendMessage targets the + * active conversation, so if the user navigated away the message stays + * queued on its own conversation until flushPendingMessage picks it up. + */ + private async sendQueuedMessage( + convId: string, + msg: { content: string; extras?: DatabaseMessageExtra[] } + ): Promise { + if (conversationsStore.activeConversation?.id !== convId) { + this.pendingMessages.set(convId, msg); + + return; + } + + await this.sendMessage(msg.content, msg.extras); + } + /** * Streams a fresh summary of `source` into the compaction node and re-stamps * the reduced context afterwards. `rollback` restores the row when nothing diff --git a/tools/ui/src/routes/(chat)/chat/[id]/+page.svelte b/tools/ui/src/routes/(chat)/chat/[id]/+page.svelte index c4a9eca2ee7f..ce6c88ba733a 100644 --- a/tools/ui/src/routes/(chat)/chat/[id]/+page.svelte +++ b/tools/ui/src/routes/(chat)/chat/[id]/+page.svelte @@ -106,6 +106,8 @@ chatStore.syncLoadingStateForChat(chatId); // server probe (with localStorage fallback) and attach await chatStore.discoverActiveStream(chatId); + // a message queued here may have finished its turn while another chat was open + await chatStore.flushPendingMessage(chatId); if ((qParam !== null || modelParam !== null) && !urlParamsProcessed) { await handleUrlParams(); diff --git a/tools/ui/tests/unit/queued-message-isolation.test.ts b/tools/ui/tests/unit/queued-message-isolation.test.ts new file mode 100644 index 000000000000..32e0f9d68dd5 --- /dev/null +++ b/tools/ui/tests/unit/queued-message-isolation.test.ts @@ -0,0 +1,81 @@ +// A message queued (steering or pending) while a conversation is busy belongs +// to that conversation: when its turn ends while another chat is open, it must +// stay queued there instead of being sent into the chat the user is viewing. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('$lib/services/database.service', () => ({ + DatabaseService: { + getConversation: vi.fn(), + getConversationMessages: vi.fn(), + updateConversation: vi.fn(), + updateCurrentNode: vi.fn() + } +})); + +import { DatabaseService } from '$lib/services/database.service'; +import { chatStore } from '$lib/stores/chat/index.svelte'; +import { conversationsStore } from '$lib/stores/conversations/index.svelte'; + +type QueuedMessage = { content: string }; +type QueueAccess = { sendQueuedMessage(convId: string, msg: QueuedMessage): Promise }; + +const sendQueuedMessage = (convId: string, msg: QueuedMessage) => + (chatStore as unknown as QueueAccess).sendQueuedMessage(convId, msg); + +async function openConversation(id: string) { + vi.mocked(DatabaseService.getConversation).mockResolvedValue({ + currNode: null, + id, + lastModified: 1000, + name: id + }); + vi.mocked(DatabaseService.getConversationMessages).mockResolvedValue([]); + + expect(await conversationsStore.loadConversation(id)).toBe(true); +} + +beforeEach(() => { + vi.restoreAllMocks(); + chatStore.clearPendingMessage('conv-a'); + chatStore.clearPendingMessage('conv-b'); +}); + +describe('queued message isolation', () => { + it('keeps the message on its own conversation when another one is active', async () => { + const send = vi.spyOn(chatStore, 'sendMessage').mockResolvedValue(); + + await openConversation('conv-b'); + await sendQueuedMessage('conv-a', { content: 'steer a' }); + + expect(send).not.toHaveBeenCalled(); + expect(chatStore.getPendingMessageContent('conv-a')).toBe('steer a'); + expect(chatStore.getPendingMessageContent('conv-b')).toBeNull(); + }); + + it('sends right away when its conversation is still active', async () => { + const send = vi.spyOn(chatStore, 'sendMessage').mockResolvedValue(); + + await openConversation('conv-a'); + await sendQueuedMessage('conv-a', { content: 'steer a' }); + + expect(send).toHaveBeenCalledWith('steer a', undefined); + expect(chatStore.hasPendingMessage('conv-a')).toBe(false); + }); + + it('flushes the left-behind message only once its conversation is active again', async () => { + const send = vi.spyOn(chatStore, 'sendMessage').mockResolvedValue(); + + chatStore.injectPendingMessage('conv-a', 'steer a'); + await openConversation('conv-b'); + await chatStore.flushPendingMessage('conv-a'); + + expect(send).not.toHaveBeenCalled(); + + await openConversation('conv-a'); + await chatStore.flushPendingMessage('conv-a'); + + expect(send).toHaveBeenCalledWith('steer a', undefined); + expect(chatStore.hasPendingMessage('conv-a')).toBe(false); + }); +}); From 23426d797e4fc3d50a56152fb04acf811966679b Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 2 Nov 2026 12:35:25 +0100 Subject: [PATCH 25/36] ui: align alert dialog styling with other overlay surfaces (#31312) Use the popover background, softer border, rounded-xl corners and shadow-md like dropdowns, hover cards and the settings dialog, and match the dialog title line height. Assisted-by: Claude (Opus 5.5 Medium) --- .../components/ui/alert-dialog/alert-dialog-content.svelte | 6 +++--- .../components/ui/alert-dialog/alert-dialog-title.svelte | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/ui/src/lib/components/ui/alert-dialog/alert-dialog-content.svelte b/tools/ui/src/lib/components/ui/alert-dialog/alert-dialog-content.svelte index a9a321437819..496c2dbe359e 100644 --- a/tools/ui/src/lib/components/ui/alert-dialog/alert-dialog-content.svelte +++ b/tools/ui/src/lib/components/ui/alert-dialog/alert-dialog-content.svelte @@ -19,13 +19,13 @@ From 5c2ea1ab11a17a750dabffe31e8c182ddf4966cc Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 2 Nov 2026 12:36:00 +0100 Subject: [PATCH 26/36] ui: disable model deletion the server would refuse (#31305) The router only removes models it took from the Hugging Face cache, but the models manager offered "Delete from disk" for every model, so a model from --models-preset or --models-dir failed with a "not removable" toast. Read the source and can_remove fields of /models and show the delete action disabled for those models, with a tooltip (a hint line in the mobile drawer) saying how to remove them instead. Assisted-by: Claude (Opus 5.5 Medium) --- .../ModelsManagerModelConfiguration.svelte | 48 ++++++++++--- .../app/models/ModelsManager/row-actions.ts | 5 ++ .../app/navigation/DropdownMenuActions.svelte | 68 +++++++++++++------ .../lib/constants/models-manager.constants.ts | 20 +++++- tools/ui/src/lib/enums/index.ts | 7 +- tools/ui/src/lib/enums/server.enums.ts | 10 +++ .../ui/src/lib/stores/models/index.svelte.ts | 22 +++++- tools/ui/src/lib/types/api.d.ts | 5 ++ .../stories/ModelsManager.stories.svelte | 4 +- 9 files changed, 156 insertions(+), 33 deletions(-) diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte index 0bf3ed49140d..427ab0189e20 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte @@ -7,6 +7,7 @@ import { Trash2 } from '@lucide/svelte'; import { DialogConfirmDownload } from '$lib/components/app/dialogs'; import { Button } from '$lib/components/ui/button'; + import * as Tooltip from '$lib/components/ui/tooltip'; import { ModelDownloadConfirmAction, ServerModelStatus } from '$lib/enums'; import { HuggingFaceService } from '$lib/services'; import { deviceStore, modelsStore, uiStore } from '$lib/stores'; @@ -26,6 +27,8 @@ // the same removal the table row offers, confirmed by the same dialog let deleteOpen = $state(false); + // a model the router did not take from its HF cache is not its own to delete + let removeBlockedReason = $derived(modelsStore.removeBlockedReason(option.model)); // the props cache is a plain Map, so the read has to name its version to stay reactive let serverProps = $derived.by(() => { @@ -130,15 +133,29 @@
- + {#if removeBlockedReason} + + + + {#snippet child({ props })} +
+ {@render deleteButton()} +
+ {/snippet} + + + +

{removeBlockedReason}

+ + + + + {#if deviceStore.isMobile} +

{removeBlockedReason}

+ {/if} + {:else} + {@render deleteButton()} + {/if}
{/if}
@@ -178,3 +195,16 @@ repoWithTag={option.model} /> + +{#snippet deleteButton()} + +{/snippet} diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts b/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts index f3c4363c7c6d..42c282e05c5d 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts +++ b/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts @@ -49,6 +49,9 @@ export function modelRowActions( ]; } + // a model the router did not take from its HF cache is not its own to delete + const removeBlockedReason = modelsStore.removeBlockedReason(option.model); + return [ { icon: favorite ? HeartOff : Heart, @@ -56,10 +59,12 @@ export function modelRowActions( onclick: () => modelsStore.toggleFavorite(option.model) }, { + disabled: removeBlockedReason !== null, icon: Trash2, label: 'Delete from disk', onclick: () => onDelete(option), separator: true, + tooltip: removeBlockedReason ?? undefined, variant: 'destructive' as const }, { diff --git a/tools/ui/src/lib/components/app/navigation/DropdownMenuActions.svelte b/tools/ui/src/lib/components/app/navigation/DropdownMenuActions.svelte index 161462a04be1..f3143050693e 100644 --- a/tools/ui/src/lib/components/app/navigation/DropdownMenuActions.svelte +++ b/tools/ui/src/lib/components/app/navigation/DropdownMenuActions.svelte @@ -13,6 +13,8 @@ onclick: (event: Event) => void; variant?: 'default' | 'destructive'; disabled?: boolean; + /** Shown on hover, e.g. why a disabled action is not available. */ + tooltip?: string; shortcut?: string[]; separator?: boolean; } @@ -88,7 +90,14 @@ > {@render iconComponent(action.icon, 'h-4 w-4 shrink-0')} - {action.label} + + {action.label} + + + {#if action.tooltip} + {action.tooltip} + {/if} + {#if action.shortcut} @@ -131,29 +140,50 @@ {/if} - -
- {@render iconComponent( - action.icon, - `h-4 w-4 ${action.variant === 'destructive' ? 'text-destructive' : ''}` - )} - {action.label} -
- - {#if action.shortcut} - - {/if} - + {#if action.tooltip} + + + + {#snippet child({ props })} +
+ {@render menuItem(action)} +
+ {/snippet} + + + +

{action.tooltip}

+ + + {:else} + {@render menuItem(action)} + {/if} {/each} {/if} +{#snippet menuItem(action: ActionItem)} + +
+ {@render iconComponent( + action.icon, + `h-4 w-4 ${action.variant === 'destructive' ? 'text-destructive' : ''}` + )} + {action.label} +
+ + {#if action.shortcut} + + {/if} + +{/snippet} + {#snippet iconComponent(IconComponent: Component, className: string)} {/snippet} diff --git a/tools/ui/src/lib/constants/models-manager.constants.ts b/tools/ui/src/lib/constants/models-manager.constants.ts index bbcf912dd55e..e6165f8423f6 100644 --- a/tools/ui/src/lib/constants/models-manager.constants.ts +++ b/tools/ui/src/lib/constants/models-manager.constants.ts @@ -1,6 +1,6 @@ /** Labels of the models manager table sections. */ -import { ModelsTableGroupKind } from '$lib/enums'; +import { ModelsTableGroupKind, ServerModelSource } from '$lib/enums'; export const MODELS_TABLE_GROUP_LABELS: Record = { [ModelsTableGroupKind.DOWNLOADING]: 'Downloading', @@ -16,3 +16,21 @@ export const MODELS_TABLE_GROUP_LABELS: Record = { * so the two stacked sticky rows do not show a gap or overlap by a pixel. */ export const MODELS_TABLE_GROUP_STICKY_OFFSET = 'top: calc(2.25rem - 1px)'; + +/** + * Why a model cannot be deleted from disk, by where the router found it. Only a + * model from the HF cache is the router's own to remove. + */ +export const MODEL_REMOVE_BLOCKED_REASONS: Record< + Exclude, + string +> = { + [ServerModelSource.MODELS_DIR]: + 'Found in --models-dir: delete the file from that folder to remove it.', + [ServerModelSource.PRESET]: + 'Defined in a --models-preset file: remove its section from the preset to delete it.' +}; + +/** Fallback when the router refuses removal without saying where the model came from. */ +export const MODEL_REMOVE_BLOCKED_FALLBACK = + 'The server does not allow deleting this model: it is not from the Hugging Face cache.'; diff --git a/tools/ui/src/lib/enums/index.ts b/tools/ui/src/lib/enums/index.ts index 142962826d9e..348864c742ad 100644 --- a/tools/ui/src/lib/enums/index.ts +++ b/tools/ui/src/lib/enums/index.ts @@ -86,7 +86,12 @@ export { ModelsTableSortKey } from './model.enums'; -export { ServerRole, ServerModelStatus, ServerModelsSseEventType } from './server.enums'; +export { + ServerRole, + ServerModelSource, + ServerModelStatus, + ServerModelsSseEventType +} from './server.enums'; export { ParameterSource, SyncableParameterType, SettingsFieldType } from './settings.enums'; diff --git a/tools/ui/src/lib/enums/server.enums.ts b/tools/ui/src/lib/enums/server.enums.ts index 6f9058942671..891fb172028f 100644 --- a/tools/ui/src/lib/enums/server.enums.ts +++ b/tools/ui/src/lib/enums/server.enums.ts @@ -26,6 +26,16 @@ export enum ServerModelStatus { UNLOADED = 'unloaded' } +/** + * Where the router found a model - matches server_model_source in + * tools/server/server-models.h. Only a cached model can be removed from disk. + */ +export enum ServerModelSource { + CACHE = 'cache', + MODELS_DIR = 'models_dir', + PRESET = 'preset' +} + /** * /models/sse event type enum - discriminates the records broadcast on the * model status feed in ROUTER mode. Matches the event names emitted by diff --git a/tools/ui/src/lib/stores/models/index.svelte.ts b/tools/ui/src/lib/stores/models/index.svelte.ts index 0a683c24bc1b..990bcf8e898b 100644 --- a/tools/ui/src/lib/stores/models/index.svelte.ts +++ b/tools/ui/src/lib/stores/models/index.svelte.ts @@ -12,10 +12,12 @@ import { FAVORITE_MODELS_LOCALSTORAGE_KEY, HIDDEN_MODELS_LOCALSTORAGE_KEY, MODEL_GROUP_OPEN_LOCALSTORAGE_KEY, + MODEL_REMOVE_BLOCKED_FALLBACK, + MODEL_REMOVE_BLOCKED_REASONS, RECENT_MODEL_LIMIT, RECENT_MODELS_LOCALSTORAGE_KEY } from '$lib/constants'; -import { ReasoningEffort, ServerModelStatus } from '$lib/enums'; +import { ReasoningEffort, ServerModelSource, ServerModelStatus } from '$lib/enums'; import { HuggingFaceService } from '$lib/services/huggingface.service'; import { ModelsService } from '$lib/services/models.service'; // direct imports between stores, not via the barrel, to avoid circular deps @@ -23,6 +25,7 @@ import { conversationsStore } from '$lib/stores/conversations/index.svelte'; import { type ModelPropsHost, ModelPropsManager } from '$lib/stores/models/props.svelte'; import { type ModelStatusHost, ModelStatusManager } from '$lib/stores/models/status.svelte'; import { serverStore } from '$lib/stores/server.svelte'; +import type { ApiModelDataEntry } from '$lib/types'; import type { ParsedModelId } from '$lib/types/models'; import { getConversationModel } from '$lib/utils/conversation-utils'; import { repoOf } from '$lib/utils/model-names'; @@ -424,6 +427,23 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { return this.isModelLoaded(modelId) && !this._status.isOperationInProgress(modelId); } + /** + * Why the router would refuse to delete a model from disk, or null when it can. + * Only a model from the HF cache is removable; a server that does not report + * `can_remove` is let through and answers for itself. + */ + removeBlockedReason(modelId: string): string | null { + const model = this.routerModels.find((m) => m.id === modelId); + + if (!model || model.can_remove !== false) return null; + + if (model.source && model.source !== ServerModelSource.CACHE) { + return MODEL_REMOVE_BLOCKED_REASONS[model.source]; + } + + return MODEL_REMOVE_BLOCKED_FALLBACK; + } + /** * Select a model. `recordRecent` keeps automatic picks out of the recency list. */ diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.ts index 72d42a6c5b64..d3bfbb184369 100644 --- a/tools/ui/src/lib/types/api.d.ts +++ b/tools/ui/src/lib/types/api.d.ts @@ -3,6 +3,7 @@ import type { ModelSidecar } from '$lib/constants'; import type { ContentPartType, FileTypeAudio, + ServerModelSource, ServerModelsSseEventType, ServerModelStatus, ServerRole @@ -95,6 +96,10 @@ export interface ApiModelDataEntry { path: string; /** Current status of the model */ status: ApiModelStatus; + /** Where the router found the model: the HF cache, --models-dir or a preset */ + source?: ServerModelSource; + /** Whether the router can delete the model's files (only cached models) */ + can_remove?: boolean; /** Alternative names that resolve to this model */ aliases?: string[]; /** Informational tags for this model */ diff --git a/tools/ui/tests/stories/ModelsManager.stories.svelte b/tools/ui/tests/stories/ModelsManager.stories.svelte index dd474cee5ed8..77e53f8e3b1f 100644 --- a/tools/ui/tests/stories/ModelsManager.stories.svelte +++ b/tools/ui/tests/stories/ModelsManager.stories.svelte @@ -6,7 +6,7 @@ ModelsTableGroup } from '$lib/components/app/models/ModelsManager/utils'; import { MODELS_TABLE_GROUP_LABELS } from '$lib/constants'; - import { ModelGroupKind, ModelsTableGroupKind } from '$lib/enums'; + import { ModelGroupKind, ModelsTableGroupKind, ServerModelStatus } from '$lib/enums'; import { modelsStore } from '$lib/stores/models/index.svelte'; import type { ModelOption } from '$lib/types/models'; @@ -60,7 +60,7 @@ object: 'model', owned_by: 'llamacpp', path: `/models/${qwen.model}`, - status: { value: 'loaded' } + status: { value: ServerModelStatus.LOADED } } ]; From cbbd7c240fe78f4abc23593baa6b4b84ca85f10d Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sat, 31 Oct 2026 23:23:52 +0800 Subject: [PATCH 27/36] ci: update to ubuntu26.04 Signed-off-by: Aaron Teo --- .github/workflows/build-3rd-party.yml | 4 +-- .../workflows/build-and-test-snapdragon.yml | 6 ++-- .github/workflows/build-android.yml | 6 ++-- .github/workflows/build-cache.yml | 2 +- .github/workflows/build-cpu.yml | 11 ++------ .github/workflows/build-cross.yml | 6 ++-- .github/workflows/build-sanitize.yml | 4 +-- .github/workflows/build-sycl.yml | 4 +-- .github/workflows/build-virtgpu.yml | 2 +- .github/workflows/build-vulkan.yml | 4 +-- .github/workflows/build-wasi.yml | 2 +- .github/workflows/build-wasm.yml | 2 +- .github/workflows/build-webgpu.yml | 2 +- .github/workflows/copilot-setup-steps.yml | 2 +- .github/workflows/docker.yml | 28 +++++++++---------- .github/workflows/gguf-publish.yml | 2 +- .github/workflows/make-release.yml | 2 +- .../workflows/python-check-requirements.yml | 2 +- .github/workflows/release-publish.yml | 2 +- .github/workflows/release.yml | 10 +++++-- .github/workflows/server.yml | 2 +- .github/workflows/ui.yml | 4 +-- .github/workflows/winget.yml | 2 +- 23 files changed, 54 insertions(+), 57 deletions(-) diff --git a/.github/workflows/build-3rd-party.yml b/.github/workflows/build-3rd-party.yml index fe25f3909957..0b9e52647add 100644 --- a/.github/workflows/build-3rd-party.yml +++ b/.github/workflows/build-3rd-party.yml @@ -31,8 +31,8 @@ env: LLAMA_ARG_LOG_TIMESTAMPS: 1 jobs: - ubuntu-24-llguidance: - runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + ubuntu-26-llguidance: + runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }} steps: - name: Clone diff --git a/.github/workflows/build-and-test-snapdragon.yml b/.github/workflows/build-and-test-snapdragon.yml index f7f94d7ee996..a95f64acdb22 100644 --- a/.github/workflows/build-and-test-snapdragon.yml +++ b/.github/workflows/build-and-test-snapdragon.yml @@ -33,7 +33,7 @@ concurrency: jobs: android-ndk-snapdragon: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 container: image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.8' defaults: @@ -67,7 +67,7 @@ jobs: path: pkg-snapdragon/llama.cpp linux-iot-snapdragon: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 container: image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.8' defaults: @@ -103,7 +103,7 @@ jobs: test-snapdragon-qdc: name: Test on QDC Device (${{ matrix.device }}) needs: [android-ndk-snapdragon, linux-iot-snapdragon] - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-26.04-arm timeout-minutes: 180 strategy: fail-fast: false diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml index 4db7c2269044..3923891fff26 100644 --- a/.github/workflows/build-android.yml +++ b/.github/workflows/build-android.yml @@ -37,7 +37,7 @@ env: jobs: default: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Clone @@ -63,7 +63,7 @@ jobs: ./gradlew build --no-daemon ndk: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 container: image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.3' defaults: @@ -101,7 +101,7 @@ jobs: path: pkg-adb/llama.cpp arm64: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 env: NDK_VERSION: "29.0.14206865" diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml index 72442ae95863..59630dff907a 100644 --- a/.github/workflows/build-cache.yml +++ b/.github/workflows/build-cache.yml @@ -41,7 +41,7 @@ jobs: # version: ${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }} ubuntu-24-openvino-cache: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 env: # Sync versions in build-openvino.yml, ci-self-hosted-openvino.yml, release.yml, release-publish.yml, build-cache.yml, .devops/openvino.Dockerfile, docs/backend/OPENVINO.md diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.yml index 76a030e67d35..81c9405ecef5 100644 --- a/.github/workflows/build-cpu.yml +++ b/.github/workflows/build-cpu.yml @@ -55,9 +55,9 @@ jobs: matrix: include: - build: 'x64' - os: ubuntu-22.04 + os: ubuntu-26.04 - build: 'arm64' - os: ubuntu-24.04-arm + os: ubuntu-26.04-arm runs-on: ${{ matrix.os }} @@ -81,13 +81,6 @@ jobs: libjpeg-dev build-essential libssl-dev \ git-lfs - - name: Toolchain workaround (GCC 14) - if: ${{ contains(matrix.os, 'ubuntu-24.04') }} - run: | - sudo apt-get install -y gcc-14 g++-14 - echo "CC=gcc-14" >> "$GITHUB_ENV" - echo "CXX=g++-14" >> "$GITHUB_ENV" - - name: Python Dependencies id: python_depends run: | diff --git a/.github/workflows/build-cross.yml b/.github/workflows/build-cross.yml index cc432bd4f419..61ea0dcd46f2 100644 --- a/.github/workflows/build-cross.yml +++ b/.github/workflows/build-cross.yml @@ -163,7 +163,7 @@ jobs: # cmake --build build --config Release -j $(nproc) debian-13-loongarch64-cpu-cross: - runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }} container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671 steps: @@ -218,7 +218,7 @@ jobs: cmake --build build --config Release -j $(nproc) debian-13-loongarch64-vulkan-cross: - runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }} container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671 steps: @@ -277,7 +277,7 @@ jobs: cmake --build build --config Release -j $(nproc) ubuntu-24-riscv64-cpu-spacemit-ime-cross: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 env: # Make sure this is in sync with build-cache.yml diff --git a/.github/workflows/build-sanitize.yml b/.github/workflows/build-sanitize.yml index ed75a2dc1cee..c9007e5871d0 100644 --- a/.github/workflows/build-sanitize.yml +++ b/.github/workflows/build-sanitize.yml @@ -45,9 +45,9 @@ jobs: include: # thread and address doesn't run properly on some self hosted machines, so run it on Github instead - sanitizer: ADDRESS - machine: ubuntu-24.04 + machine: ubuntu-26.04 - sanitizer: THREAD - machine: ubuntu-24.04 + machine: ubuntu-26.04 - sanitizer: UNDEFINED machine: [self-hosted, X64, Linux, CPU] diff --git a/.github/workflows/build-sycl.yml b/.github/workflows/build-sycl.yml index ecfab4e52403..8c1a35ede884 100644 --- a/.github/workflows/build-sycl.yml +++ b/.github/workflows/build-sycl.yml @@ -38,7 +38,7 @@ env: LLAMA_ARG_LOG_TIMESTAMPS: 1 jobs: - ubuntu-24-sycl: + ubuntu-26-sycl: strategy: matrix: build: [fp32, fp16] @@ -48,7 +48,7 @@ jobs: - build: fp16 fp16: ON - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 env: ONEAPI_ROOT: /opt/intel/oneapi/ diff --git a/.github/workflows/build-virtgpu.yml b/.github/workflows/build-virtgpu.yml index 57686578304f..9b3838b7015a 100644 --- a/.github/workflows/build-virtgpu.yml +++ b/.github/workflows/build-virtgpu.yml @@ -32,7 +32,7 @@ concurrency: jobs: ubuntu-24-virtgpu: - runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }} steps: - name: Clone diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml index c4c08621f958..05732ca97963 100644 --- a/.github/workflows/build-vulkan.yml +++ b/.github/workflows/build-vulkan.yml @@ -42,7 +42,7 @@ env: jobs: ubuntu-arm64: - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-26.04-arm steps: - name: Clone @@ -99,7 +99,7 @@ jobs: save: true ubuntu-llvmpipe: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 steps: - name: Clone diff --git a/.github/workflows/build-wasi.yml b/.github/workflows/build-wasi.yml index c6dbd66665bc..8abe8d8d563f 100644 --- a/.github/workflows/build-wasi.yml +++ b/.github/workflows/build-wasi.yml @@ -49,7 +49,7 @@ env: jobs: ubuntu-webgpu: - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-26.04-arm steps: - name: Clone diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index f96fd85a914a..2ca33332373d 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -50,7 +50,7 @@ env: jobs: ubuntu-webgpu: - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-26.04-arm steps: - name: Clone diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.yml index 88f2c6f5dfa2..4e9b8588d031 100644 --- a/.github/workflows/build-webgpu.yml +++ b/.github/workflows/build-webgpu.yml @@ -34,7 +34,7 @@ env: jobs: format: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 steps: - name: Clone diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 5527e73b6f3d..9eb1113abf82 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -16,7 +16,7 @@ cache-mode: none jobs: # The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot. copilot-setup-steps: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 # Set the permissions to the lowest permissions possible needed for your steps. # Copilot will be given its own token for its operations. diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 75b3343095a9..fcae6c36bbdc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -70,7 +70,7 @@ jobs: prepare_matrices: name: Prepare Docker matrices - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 outputs: build_matrix: ${{ steps.matrices.outputs.build_matrix }} merge_matrix: ${{ steps.matrices.outputs.merge_matrix }} @@ -87,19 +87,19 @@ jobs: # Keep all build targets in one place and derive merge targets from it. cat > build-matrix.json <<'JSON' [ - { "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" }, - { "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-arm" }, - { "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-s390x", "prebuilt_ui": true }, - { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, - { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" }, - { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, - { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" }, - { "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, - { "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, - { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" }, - { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-arm" }, - { "tag": "rocm", "dockerfile": ".devops/rocm.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, - { "tag": "openvino", "dockerfile": ".devops/openvino.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" } + { "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04" }, + { "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04-arm" }, + { "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04-s390x", "prebuilt_ui": true }, + { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" }, + { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04-arm" }, + { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" }, + { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04-arm" }, + { "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" }, + { "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" }, + { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04" }, + { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04-arm" }, + { "tag": "rocm", "dockerfile": ".devops/rocm.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-26.04" }, + { "tag": "openvino", "dockerfile": ".devops/openvino.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-26.04" } ] JSON diff --git a/.github/workflows/gguf-publish.yml b/.github/workflows/gguf-publish.yml index 367b29c60f94..94a49146681e 100644 --- a/.github/workflows/gguf-publish.yml +++ b/.github/workflows/gguf-publish.yml @@ -24,7 +24,7 @@ permissions: jobs: deploy: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/make-release.yml b/.github/workflows/make-release.yml index 2e5e44caaab8..1f5d43755e78 100644 --- a/.github/workflows/make-release.yml +++ b/.github/workflows/make-release.yml @@ -39,7 +39,7 @@ permissions: jobs: make-release: - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Checkout diff --git a/.github/workflows/python-check-requirements.yml b/.github/workflows/python-check-requirements.yml index 9738ff778920..464cc94e1c84 100644 --- a/.github/workflows/python-check-requirements.yml +++ b/.github/workflows/python-check-requirements.yml @@ -24,7 +24,7 @@ concurrency: jobs: python-check-requirements: - runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }} name: check-requirements steps: - name: Check out source repository diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index 243b870e6664..959df617e769 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -32,7 +32,7 @@ jobs: id-token: write attestations: write - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 outputs: should_release: ${{ steps.check.outputs.should_release }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index acda37fb9cfc..4bc599f82060 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -176,8 +176,12 @@ jobs: include: - build: 'x64' os: ubuntu-22.04 + - build: '26.04-x64' + os: ubuntu-26.04 - build: 'arm64' os: ubuntu-24.04-arm + - build: '26.04-arm64' + os: ubuntu-26.04-arm - build: 's390x' os: ubuntu-24.04-s390x - build: '26.04-s390x' @@ -463,7 +467,7 @@ jobs: needs: [check-release, ui-build] if: ${{ needs.check-release.outputs.should_release == 'true' }} - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 env: NDK_VERSION: "29.0.14206865" @@ -548,7 +552,7 @@ jobs: needs: [check-release, ui-build] if: ${{ needs.check-release.outputs.should_release == 'true' }} - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 container: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.8' defaults: @@ -601,7 +605,7 @@ jobs: needs: [check-release, ui-build] if: ${{ needs.check-release.outputs.should_release == 'true' }} - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 container: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.8' defaults: diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 05dd0ac4595a..61eb9217095c 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -59,7 +59,7 @@ concurrency: jobs: ubuntu: - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-26.04-arm steps: - name: Dependencies diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index c5c6a32a8177..af289b56b152 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -51,7 +51,7 @@ jobs: ui-checks: name: Checks needs: ui-build - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 continue-on-error: true steps: - name: Checkout code @@ -110,7 +110,7 @@ jobs: e2e-tests: name: E2E Tests needs: ui-build - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 steps: - name: Checkout code uses: actions/checkout@v6 diff --git a/.github/workflows/winget.yml b/.github/workflows/winget.yml index 8b5bd355c16e..1f0f24905d06 100644 --- a/.github/workflows/winget.yml +++ b/.github/workflows/winget.yml @@ -12,7 +12,7 @@ permissions: jobs: update: name: Update Winget Package - runs-on: ubuntu-24.04 # previously ubuntu-latest + runs-on: ubuntu-26.04 if: github.repository_owner == 'ggml-org' steps: From 09264042e9ff19ecc3fb6e236d69ada7ddb35ae8 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sat, 31 Oct 2026 23:52:45 +0800 Subject: [PATCH 28/36] ci: bring back toolchain workaround Signed-off-by: Aaron Teo --- .github/workflows/release.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4bc599f82060..2b1a366c9f11 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -178,12 +178,8 @@ jobs: os: ubuntu-22.04 - build: '26.04-x64' os: ubuntu-26.04 - - build: 'arm64' - os: ubuntu-24.04-arm - build: '26.04-arm64' os: ubuntu-26.04-arm - - build: 's390x' - os: ubuntu-24.04-s390x - build: '26.04-s390x' os: ubuntu-26.04-s390x @@ -209,7 +205,7 @@ jobs: sudo apt-get install -y build-essential libssl-dev - name: Toolchain workaround (GCC 14) - if: ${{ contains(matrix.os, 'ubuntu-24.04') }} + if: ${{ !contains(matrix.os, 'ubuntu-26.04') }} run: | sudo apt-get install -y gcc-14 g++-14 echo "CC=gcc-14" >> "$GITHUB_ENV" @@ -263,7 +259,7 @@ jobs: - build: 'x64' os: ubuntu-22.04 - build: 'arm64' - os: ubuntu-24.04-arm + os: ubuntu-26.04-arm runs-on: ${{ matrix.os }} @@ -290,9 +286,7 @@ jobs: sudo apt-get install -y build-essential mesa-vulkan-drivers vulkan-sdk libssl-dev else sudo apt-get update -y - sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build - echo "CC=gcc-14" >> "$GITHUB_ENV" - echo "CXX=g++-14" >> "$GITHUB_ENV" + sudo apt-get install -y build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build fi - name: ccache From c5c23538fe1ec32ac4684ff0c45d53eec3bf0792 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sat, 31 Oct 2026 23:58:53 +0800 Subject: [PATCH 29/36] ci: remove workaround completely Signed-off-by: Aaron Teo --- .github/workflows/release.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2b1a366c9f11..7b41188cd584 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -204,13 +204,6 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential libssl-dev - - name: Toolchain workaround (GCC 14) - if: ${{ !contains(matrix.os, 'ubuntu-26.04') }} - run: | - sudo apt-get install -y gcc-14 g++-14 - echo "CC=gcc-14" >> "$GITHUB_ENV" - echo "CXX=g++-14" >> "$GITHUB_ENV" - - name: ccache if: ${{ !contains(matrix.os, 's390x') }} uses: ggml-org/ccache-action@v1.2.24 From 11ead62f4d3a1ea132ceaf5c7abc171442edfbc1 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sun, 1 Nov 2026 00:01:13 +0800 Subject: [PATCH 30/36] ci: more switching to ubuntu 26.04 Signed-off-by: Aaron Teo --- .devops/cpu.Dockerfile | 22 +++++++++++++--------- .github/workflows/build-cross.yml | 2 +- .github/workflows/build-sycl.yml | 4 ++-- .github/workflows/build-virtgpu.yml | 2 +- .github/workflows/build-vulkan.yml | 12 +++++------- .github/workflows/build-wasi.yml | 4 ++-- .github/workflows/build-wasm.yml | 4 ++-- .github/workflows/build-webgpu.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/release-publish.yml | 6 +++--- .github/workflows/server.yml | 4 ++-- .github/workflows/ui.yml | 4 ++-- 12 files changed, 35 insertions(+), 33 deletions(-) diff --git a/.devops/cpu.Dockerfile b/.devops/cpu.Dockerfile index 0eba8f008e46..11483e65c0ee 100644 --- a/.devops/cpu.Dockerfile +++ b/.devops/cpu.Dockerfile @@ -1,4 +1,4 @@ -ARG UBUNTU_VERSION=24.04 +ARG UBUNTU_VERSION=26.04 ARG BUILD_DATE=N/A ARG APP_VERSION=N/A ARG APP_REVISION=N/A @@ -22,9 +22,7 @@ FROM docker.io/ubuntu:$UBUNTU_VERSION AS build ARG TARGETARCH RUN apt-get update && \ - apt-get install -y gcc-14 g++-14 build-essential git cmake libssl-dev - -ENV CC=gcc-14 CXX=g++-14 + apt-get install -y build-essential git cmake libssl-dev WORKDIR /app @@ -85,14 +83,20 @@ COPY --from=build /app/full /app WORKDIR /app +ENV PATH="/root/.venv/bin:/root/.local/bin:${PATH}" + +# Flag for compatibility with pip +ARG UV_INDEX_STRATEGY="unsafe-best-match" RUN apt-get update \ && apt-get install -y \ + build-essential \ + curl \ git \ - python3 \ - python3-pip \ - python3-wheel \ - && pip install --break-system-packages --upgrade setuptools \ - && pip install --break-system-packages -r requirements.txt \ + ca-certificates \ + && curl -LsSf https://astral.sh/uv/install.sh | sh \ + && uv python install 3.13 \ + && uv venv --python 3.13 /root/.venv \ + && uv pip install --python /root/.venv/bin/python -r requirements.txt \ && apt autoremove -y \ && apt clean -y \ && rm -rf /tmp/* /var/tmp/* \ diff --git a/.github/workflows/build-cross.yml b/.github/workflows/build-cross.yml index 61ea0dcd46f2..00cd5dc5ceca 100644 --- a/.github/workflows/build-cross.yml +++ b/.github/workflows/build-cross.yml @@ -276,7 +276,7 @@ jobs: cmake --build build --config Release -j $(nproc) - ubuntu-24-riscv64-cpu-spacemit-ime-cross: + ubuntu-26-riscv64-cpu-spacemit-ime-cross: runs-on: ubuntu-26.04 env: diff --git a/.github/workflows/build-sycl.yml b/.github/workflows/build-sycl.yml index 8c1a35ede884..d48ffd6eda9e 100644 --- a/.github/workflows/build-sycl.yml +++ b/.github/workflows/build-sycl.yml @@ -103,7 +103,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} with: - key: sycl-ubuntu-24-${{ matrix.build }} + key: sycl-ubuntu-26-${{ matrix.build }} folder: llama.cpp hf_bucket: ggml-org/cache @@ -131,7 +131,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: sycl-ubuntu-24-${{ matrix.build }} + key: sycl-ubuntu-26-${{ matrix.build }} folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache diff --git a/.github/workflows/build-virtgpu.yml b/.github/workflows/build-virtgpu.yml index 9b3838b7015a..6e7bf342ebf3 100644 --- a/.github/workflows/build-virtgpu.yml +++ b/.github/workflows/build-virtgpu.yml @@ -31,7 +31,7 @@ concurrency: cancel-in-progress: true jobs: - ubuntu-24-virtgpu: + ubuntu-26-virtgpu: runs-on: ${{ 'ubuntu-26.04-arm' || 'ubuntu-26.04' }} steps: diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml index 05732ca97963..d2375b0141fb 100644 --- a/.github/workflows/build-vulkan.yml +++ b/.github/workflows/build-vulkan.yml @@ -53,9 +53,7 @@ jobs: id: depends run: | sudo apt-get update - sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build - echo "CC=gcc-14" >> "$GITHUB_ENV" - echo "CXX=g++-14" >> "$GITHUB_ENV" + sudo apt-get install -y build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build - name: ccache uses: ggml-org/ccache-action@v1.2.24 @@ -69,7 +67,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} with: - key: vulkan-ubuntu-24.04-arm + key: vulkan-ubuntu-26.04-arm folder: llama.cpp hf_bucket: ggml-org/cache @@ -92,7 +90,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: vulkan-ubuntu-24.04-arm + key: vulkan-ubuntu-26.04-arm folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache @@ -137,7 +135,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} with: - key: vulkan-ubuntu-24.04-llvmpipe + key: vulkan-ubuntu-26.04-llvmpipe folder: llama.cpp hf_bucket: ggml-org/cache @@ -156,7 +154,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: vulkan-ubuntu-24.04-llvmpipe + key: vulkan-ubuntu-26.04-llvmpipe folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache diff --git a/.github/workflows/build-wasi.yml b/.github/workflows/build-wasi.yml index 8abe8d8d563f..2ea854c0160f 100644 --- a/.github/workflows/build-wasi.yml +++ b/.github/workflows/build-wasi.yml @@ -97,7 +97,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} with: - key: webgpu-ubuntu-24.04-arm-wasi + key: webgpu-ubuntu-26.04-arm-wasi folder: llama.cpp hf_bucket: ggml-org/cache @@ -135,7 +135,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: webgpu-ubuntu-24.04-arm-wasi + key: webgpu-ubuntu-26.04-arm-wasi folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 2ca33332373d..bfe57033e899 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -84,7 +84,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} with: - key: webgpu-ubuntu-24.04-arm-wasm + key: webgpu-ubuntu-26.04-arm-wasm folder: llama.cpp hf_bucket: ggml-org/cache @@ -107,7 +107,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: webgpu-ubuntu-24.04-arm-wasm + key: webgpu-ubuntu-26.04-arm-wasm folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.yml index 4e9b8588d031..ec515adcc61e 100644 --- a/.github/workflows/build-webgpu.yml +++ b/.github/workflows/build-webgpu.yml @@ -45,7 +45,7 @@ jobs: wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null sudo add-apt-repository -y \ - "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-22 main" + "deb http://apt.llvm.org/resolute/ llvm-toolchain-resolute-22 main" sudo apt-get update sudo apt-get install -y clang-format-22 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fcae6c36bbdc..e6d99a32fd90 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -400,7 +400,7 @@ jobs: merge_arch_tags: name: Create shared tags from digests needs: [prepare_matrices, push_to_registry, create_tag] - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 permissions: contents: read packages: write diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index 959df617e769..f95236e5d9fe 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -170,12 +170,12 @@ jobs: **Linux:** - [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz) - - [Ubuntu arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-arm64.tar.gz) - - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz) + - [Ubuntu 26.04 x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-26.04-x64.tar.gz) + - [Ubuntu 26.04 arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-26.04-arm64.tar.gz) - [Ubuntu 26.04 s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-26.04-s390x.tar.gz) - [Ubuntu 26.04 s390x (zDNN)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-zdnn-s390x.tar.gz) - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz) - - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz) + - [Ubuntu 26.04 arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz) - [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz) - [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 61eb9217095c..f3c577ec6d28 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -94,7 +94,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CI }} with: - key: server-ubuntu-24.04-arm + key: server-ubuntu-26.04-arm folder: llama.cpp hf_bucket: ggml-org/cache @@ -111,7 +111,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: - key: server-ubuntu-24.04-arm + key: server-ubuntu-26.04-arm folder: llama.cpp evict-old-files: 1d hf_bucket: ggml-org/cache diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index af289b56b152..c5c6a32a8177 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -51,7 +51,7 @@ jobs: ui-checks: name: Checks needs: ui-build - runs-on: ubuntu-26.04 + runs-on: ubuntu-24.04 continue-on-error: true steps: - name: Checkout code @@ -110,7 +110,7 @@ jobs: e2e-tests: name: E2E Tests needs: ui-build - runs-on: ubuntu-26.04 + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@v6 From 9643fc061870af203cf1641d39a501128cc3a59f Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sun, 1 Nov 2026 12:35:06 +0800 Subject: [PATCH 31/36] ci: pin android cmake version Signed-off-by: Aaron Teo --- .github/workflows/build-android.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml index 3923891fff26..bbe23cf3f497 100644 --- a/.github/workflows/build-android.yml +++ b/.github/workflows/build-android.yml @@ -56,6 +56,7 @@ jobs: uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4 with: log-accepted-android-sdk-licenses: false + packages: platform-tools cmake;3.31.6 - name: Build run: | From d95d9576ddbfa66475ff9a892e914826d6d9edb1 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 2 Nov 2026 03:42:10 +0800 Subject: [PATCH 32/36] ci: restore ubuntu 24.04 for s390x Signed-off-by: Aaron Teo --- .github/workflows/release.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7b41188cd584..8c6e23ea8a9a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -180,6 +180,8 @@ jobs: os: ubuntu-26.04 - build: '26.04-arm64' os: ubuntu-26.04-arm + - build: 's390x' + os: ubuntu-24.04-s390x - build: '26.04-s390x' os: ubuntu-26.04-s390x @@ -204,6 +206,13 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential libssl-dev + - name: Toolchain workaround (GCC 14) + if: ${{ contains(matrix.os, 'ubuntu-24.04') }} + run: | + sudo apt-get install -y gcc-14 g++-14 + echo "CC=gcc-14" >> "$GITHUB_ENV" + echo "CXX=g++-14" >> "$GITHUB_ENV" + - name: ccache if: ${{ !contains(matrix.os, 's390x') }} uses: ggml-org/ccache-action@v1.2.24 From 69c8d86f1c66deaf029d7c89f10ee16e55171651 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 2 Nov 2026 15:06:14 +0800 Subject: [PATCH 33/36] ci: fix missing s390x ubuntu 24.04 release link Signed-off-by: Aaron Teo --- .github/workflows/release-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index f95236e5d9fe..5e2e93ef9658 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -172,6 +172,7 @@ jobs: - [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz) - [Ubuntu 26.04 x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-26.04-x64.tar.gz) - [Ubuntu 26.04 arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-26.04-arm64.tar.gz) + - [Ubuntu 24.04 s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz) - [Ubuntu 26.04 s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-26.04-s390x.tar.gz) - [Ubuntu 26.04 s390x (zDNN)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-zdnn-s390x.tar.gz) - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz) From 5f1cc387eb547d78ee6723b2403001bdc934ec56 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 2 Nov 2026 19:51:51 +0800 Subject: [PATCH 34/36] ci: switch release-publish to ubuntu-latest Signed-off-by: Aaron Teo --- .github/workflows/release-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index 5e2e93ef9658..b48256d9b065 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -32,7 +32,7 @@ jobs: id-token: write attestations: write - runs-on: ubuntu-26.04 + runs-on: ubuntu-latest outputs: should_release: ${{ steps.check.outputs.should_release }} From 377966a10ef55f28b53bdbe71bf4d0b1f3821b46 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 2 Nov 2026 19:59:46 +0800 Subject: [PATCH 35/36] devops: revert --break-system-packages Signed-off-by: Aaron Teo --- .devops/cpu.Dockerfile | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.devops/cpu.Dockerfile b/.devops/cpu.Dockerfile index 11483e65c0ee..0809c8b7ec90 100644 --- a/.devops/cpu.Dockerfile +++ b/.devops/cpu.Dockerfile @@ -83,20 +83,14 @@ COPY --from=build /app/full /app WORKDIR /app -ENV PATH="/root/.venv/bin:/root/.local/bin:${PATH}" - -# Flag for compatibility with pip -ARG UV_INDEX_STRATEGY="unsafe-best-match" RUN apt-get update \ && apt-get install -y \ - build-essential \ - curl \ git \ - ca-certificates \ - && curl -LsSf https://astral.sh/uv/install.sh | sh \ - && uv python install 3.13 \ - && uv venv --python 3.13 /root/.venv \ - && uv pip install --python /root/.venv/bin/python -r requirements.txt \ + python3 \ + python3-pip \ + python3-wheel \ + && pip install --break-system-packages --upgrade setuptools \ + && pip install --break-system-packages -r requirements.txt \ && apt autoremove -y \ && apt clean -y \ && rm -rf /tmp/* /var/tmp/* \ From 0d51049e03e653a1cf2381a8a4fa3e9d9a3f49b4 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 2 Nov 2026 23:05:46 +0800 Subject: [PATCH 36/36] ci: bring back ubuntu 24.04 for arm release Signed-off-by: Aaron Teo --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8c6e23ea8a9a..f476d05daed9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -178,6 +178,8 @@ jobs: os: ubuntu-22.04 - build: '26.04-x64' os: ubuntu-26.04 + - build: 'arm64' + os: ubuntu-24.04-arm - build: '26.04-arm64' os: ubuntu-26.04-arm - build: 's390x'