Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions express/module/StaticModule.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -573,7 +573,7 @@ ErrorCode StaticModule::_resize(const std::vector<Express::VARP>& inputs) {
// Shape inference reads some inputs' CONTENT (e.g. the LLM logits-index
// scalar feeding a StridedSlice begin), which historically forced a
// full re-resize EVERY forward. Cache those inputs' bytes (tiny
// integer control tensors) and only force resize when one changed.
// control tensors) and only force resize when one changed.
// MNN_LLM_CONTENT_RESIZE_ALWAYS=1 restores the legacy behavior.
static const bool sAlwaysResize = [] {
const char* e = getenv("MNN_LLM_CONTENT_RESIZE_ALWAYS");
Expand All @@ -593,9 +593,16 @@ ErrorCode StaticModule::_resize(const std::vector<Express::VARP>& inputs) {
continue;
}
auto inputTensor = Utils::getTensor(inputs[i]);
if (inputTensor->getType().code == halide_type_float) {
// integer tensors drive shape computation; float inputs
// (embeddings / mask) do not feed shape inference
if (inputTensor->getType().code == halide_type_float && !mResource->mInputNeedCPU[i]) {
// Only float inputs consumed by shape inference need content comparison.
continue;
}
// Backend storage may use a different precision or reside on a device.
// Only cache plain host tensors; the input-copy path below handles conversion.
if (nullptr != TensorUtils::getDescribeOrigin(inputTensor)->getBackend() ||
0 != inputTensor->buffer().device) {
mContentCache[i].clear();
needResize = true;
continue;
}
auto size = inputTensor->usize();
Expand Down
88 changes: 88 additions & 0 deletions test/expr/ModuleShapeInfer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -205,3 +205,91 @@ class ModuleInputCopyRefresh : public MNNTestCase {
}
};
MNNTestSuiteRegister(ModuleInputCopyRefresh, "expr/ModuleInputCopyRefresh");

// Range consumes floating-point values during shape inference. In particular,
// ONNX masks may cast an integer sequence length to float before Range.
class ModuleRangeContentTest : public MNNTestCase {
public:
bool run(int precision) override {
auto executor = cloneCurrentExecutor();
ExecutorScope scope(executor);
for (int mode = 0; mode < 4; ++mode) {
const bool backendInput = mode == 3;
const bool floatInput = mode == 0 || backendInput;
auto length = _Input({1}, NCHW, floatInput ? halide_type_of<float>() : halide_type_of<int>());
length->setName("length");
auto limit = backendInput ? length : _ReduceMax(length, {}, false);
const bool floatRange = mode != 2;
if (floatRange && !backendInput) {
limit = _Cast<float>(limit);
}
auto output = floatRange ? _Range(_Scalar<float>(0), limit, _Scalar<float>(1))
: _Range(_Scalar<int>(0), limit, _Scalar<int>(1));
output->setName("range");
auto buffer = Variable::save({output});
std::shared_ptr<Module> module(Module::load({"length"}, {"range"},
reinterpret_cast<const uint8_t*>(buffer.data()), buffer.size()), Module::destroy);
if (!module) {
return false;
}
auto input = _Input({1}, NCHW, floatInput ? halide_type_of<float>() : halide_type_of<int>());
// Alternate host and backend outputs to also check cache invalidation on fallback.
std::shared_ptr<Module> producer;
if (backendInput) {
ScheduleConfig config;
BackendConfig backendConfig;
backendConfig.precision = static_cast<BackendConfig::PrecisionMode>(precision);
config.backendConfig = &backendConfig;
std::shared_ptr<Executor::RuntimeManager> runtime(
Executor::RuntimeManager::createRuntimeManager(config), Executor::RuntimeManager::destroy);
auto value = _Input({1}, NCHW);
value->setName("value");
auto positive = _Square(value);
positive->setName("positive");
auto producerBuffer = Variable::save({positive});
producer.reset(Module::load({"value"}, {"positive"},
reinterpret_cast<const uint8_t*>(producerBuffer.data()), producerBuffer.size(), runtime), Module::destroy);
if (!producer) {
return false;
}
}
int step = 0;
const std::vector<int> counts = backendInput ? std::vector<int>{4, 16, 4, 16, 16, 1, 9}
: std::vector<int>{3, 13, 3, 13, 13, 2, 5};
for (int count : counts) {
if (floatInput) {
input->writeMap<float>()[0] = count;
} else {
input->writeMap<int>()[0] = count;
}
VARP current = input;
if (backendInput && step != 0 && step != 2 && step != 6) {
input->writeMap<float>()[0] = sqrtf(static_cast<float>(count));
auto produced = producer->onForward({input});
if (produced.size() != 1) {
return false;
}
current = produced[0];
}
++step;
auto outputs = module->onForward({current});
if (outputs.size() != 1 || outputs[0]->getInfo() == nullptr ||
outputs[0]->getInfo()->dim != std::vector<int>{count}) {
MNN_ERROR("Range shape did not update: mode=%d, count=%d\n", mode, count);
return false;
}
for (int i = 0; i < count; ++i) {
if (floatRange) {
if (outputs[0]->readMap<float>()[i] != static_cast<float>(i)) {
return false;
}
} else if (outputs[0]->readMap<int>()[i] != i) {
return false;
}
}
}
}
return true;
}
};
MNNTestSuiteRegister(ModuleRangeContentTest, "expr/ModuleRangeContent");