完成作业1到4 实现英伟达和天数后端

.github/workflows/c-cpp.yml

@@ -0,0 +1,23 @@
+name: C/C++ CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: configure
+      run: ./configure
+    - name: make
+      run: make
+    - name: make check
+      run: make check
+    - name: make distcheck
+      run: make distcheck

.gitignore

@@ -31,6 +31,9 @@ cache/
 #GGUF
 *.gguf
 
+# Local model weights (download separately from Hugging Face)
+qwen/model.safetensors
+
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -87,4 +90,4 @@ htmlcov/
 # Windows
 Thumbs.db
 ehthumbs.db
-desktop.ini
+desktop.ini

README_ZN.md

@@ -361,6 +361,31 @@ python test/test_infer.py --model [dir_path/to/model] --test --device nvidia
 
 提交并推送你的更改。你应该看到作业#4的自动测试通过了。
 
+### 天数智芯 CoreX 环境复现
+
+天数智芯平台使用独立的 `iluvatar-gpu` 构建选项。CoreX 4.4.0 的 CUDA
+兼容编译器是 Clang Iluvatar 后端，目标架构为 `ivcore11`，不能使用 NVIDIA
+的 `compute_xx` 参数。
+
+```bash
+export COREX_HOME=/usr/local/corex
+export PATH=$COREX_HOME/bin:$PATH
+export LD_LIBRARY_PATH=$COREX_HOME/lib64:${LD_LIBRARY_PATH:-}
+export XMAKE_ROOT=y  # 仅 root 容器需要
+
+xmake f --iluvatar-gpu=y -cv
+xmake
+xmake install
+pip install ./python/
+
+ixsmi
+python test/test_runtime.py --device iluvatar
+python test/ops/add.py --device iluvatar
+```
+
+验证环境：Iluvatar BI-V150（32 GiB），CoreX SDK/Driver 4.4.0。Python 接口
+使用独立的 `iluvatar` 设备名称，与 NVIDIA 后端明确区分。
+
 ## 作业提交要求
 
 将作业代码以 Pull Request 的形式提交到 [wooway777/llaisys-26s](https://github.com/wooway777/llaisys-26s)。

include/llaisys.h

@@ -24,6 +24,7 @@ typedef enum {
     LLAISYS_DEVICE_CPU = 0,
     //// TODO: Add more device types here. Numbers need to be consecutive.
     LLAISYS_DEVICE_NVIDIA = 1,
+    LLAISYS_DEVICE_ILUVATAR = 2,
     LLAISYS_DEVICE_TYPE_COUNT
 } llaisysDeviceType_t;
 

include/llaisys/models/qwen2.h

@@ -37,6 +37,16 @@ __C {
 
     __export struct LlaisysQwen2Weights *llaisysQwen2ModelWeights(struct LlaisysQwen2Model * model);
 
+    __export void llaisysQwen2ModelLoadWeight(
+        struct LlaisysQwen2Model *model,
+        const char *name,
+        const size_t *shape,
+        size_t ndim,
+        llaisysDataType_t dtype,
+        const void *data);
+
+    __export void llaisysQwen2ModelResetCache(struct LlaisysQwen2Model *model);
+
     __export int64_t llaisysQwen2ModelInfer(struct LlaisysQwen2Model * model, int64_t * token_ids, size_t ntoken);
 }
 #endif // LLAISYS_MODELS_QWEN2_H

python/llaisys/libllaisys/__init__.py

@@ -31,6 +31,20 @@ def load_shared_library():
     if not os.path.isfile(lib_path):
         raise FileNotFoundError(f"Shared library not found: {lib_path}")
 
+    # CUDA kernels are built as a separate shared library so nvcc can perform
+    # device linking. Load it globally before the main C API library.
+    if sys.platform.startswith("linux"):
+        corex_home = os.environ.get("COREX_HOME", "/usr/local/corex")
+        corex_cudart = Path(corex_home) / "lib64" / "libcudart.so"
+        if corex_cudart.is_file():
+            ctypes.CDLL(str(corex_cudart), mode=ctypes.RTLD_GLOBAL)
+        cuda_lib = lib_dir / "libllaisys-ops-nvidia.so"
+        if cuda_lib.is_file():
+            ctypes.CDLL(str(cuda_lib), mode=ctypes.RTLD_GLOBAL)
+        else:
+            # CoreX links GPU kernels into the main library as a static archive.
+            pass
+
     return ctypes.CDLL(str(lib_path))
 
 

python/llaisys/libllaisys/llaisys_types.py

@@ -6,7 +6,8 @@
 class DeviceType(IntEnum):
     CPU = 0
     NVIDIA = 1
-    COUNT = 2
+    ILUVATAR = 2
+    COUNT = 3
 
 
 llaisysDeviceType_t = ctypes.c_int

python/llaisys/models/qwen2.py

@@ -1,23 +1,90 @@
 from typing import Sequence
 from ..libllaisys import LIB_LLAISYS
 from ..libllaisys import DeviceType
+from ..libllaisys import DataType
 
 from pathlib import Path
+import ctypes
+import json
+import torch
 import safetensors
 
+class _Qwen2Meta(ctypes.Structure):
+    _fields_ = [
+        ("dtype", ctypes.c_int), ("nlayer", ctypes.c_size_t),
+        ("hs", ctypes.c_size_t), ("nh", ctypes.c_size_t),
+        ("nkvh", ctypes.c_size_t), ("dh", ctypes.c_size_t),
+        ("di", ctypes.c_size_t), ("maxseq", ctypes.c_size_t),
+        ("voc", ctypes.c_size_t), ("epsilon", ctypes.c_float),
+        ("theta", ctypes.c_float), ("end_token", ctypes.c_int64),
+    ]
+
+
+def _dtype(dtype):
+    name = str(dtype).lower()
+    if name in ("bfloat16", "bf16"):
+        return DataType.BF16
+    if name == "float16":
+        return DataType.F16
+    if name == "float32":
+        return DataType.F32
+    if name == "float64":
+        return DataType.F64
+    if name == "int64":
+        return DataType.I64
+    raise TypeError(f"Unsupported Qwen2 weight dtype: {dtype}")
+
+
+LIB_LLAISYS.llaisysQwen2ModelCreate.argtypes = [
+    ctypes.POINTER(_Qwen2Meta), ctypes.c_int, ctypes.POINTER(ctypes.c_int), ctypes.c_int
+]
+LIB_LLAISYS.llaisysQwen2ModelCreate.restype = ctypes.c_void_p
+LIB_LLAISYS.llaisysQwen2ModelDestroy.argtypes = [ctypes.c_void_p]
+LIB_LLAISYS.llaisysQwen2ModelDestroy.restype = None
+LIB_LLAISYS.llaisysQwen2ModelLoadWeight.argtypes = [
+    ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_size_t), ctypes.c_size_t,
+    ctypes.c_int, ctypes.c_void_p
+]
+LIB_LLAISYS.llaisysQwen2ModelLoadWeight.restype = None
+LIB_LLAISYS.llaisysQwen2ModelResetCache.argtypes = [ctypes.c_void_p]
+LIB_LLAISYS.llaisysQwen2ModelResetCache.restype = None
+LIB_LLAISYS.llaisysQwen2ModelInfer.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int64), ctypes.c_size_t]
+LIB_LLAISYS.llaisysQwen2ModelInfer.restype = ctypes.c_int64
 
 class Qwen2:
 
     def __init__(self, model_path, device: DeviceType = DeviceType.CPU):
-        # TODO: Implement model constructor
-
         model_path = Path(model_path)
+        with open(model_path / "config.json", "r", encoding="utf-8") as f:
+            config = json.load(f)
+
+        hidden = int(config["hidden_size"])
+        heads = int(config["num_attention_heads"])
+        kv_heads = int(config.get("num_key_value_heads", heads))
+        self._meta = _Qwen2Meta(
+            int(DataType.BF16 if config.get("torch_dtype") == "bfloat16" else DataType.F32),
+            int(config["num_hidden_layers"]), hidden, heads, kv_heads,
+            int(config.get("head_dim", hidden // heads)), int(config["intermediate_size"]),
+            int(config["max_position_embeddings"]), int(config["vocab_size"]),
+            float(config.get("rms_norm_eps", 1e-6)), float(config.get("rope_theta", 10000.0)),
+            int(config.get("eos_token_id", 0)),
+        )
+        self.device = device
+        device_id = ctypes.c_int(0)
+        self._model = LIB_LLAISYS.llaisysQwen2ModelCreate(
+            ctypes.byref(self._meta), int(device), ctypes.byref(device_id), 1
+        )
 
         for file in sorted(model_path.glob("*.safetensors")):
-            data_ = safetensors.safe_open(file, framework="numpy", device="cpu")
+            data_ = safetensors.safe_open(file, framework="pt", device="cpu")
             for name_ in data_.keys():
-                ## TODO: load the model weights
-                pass
+                tensor = data_.get_tensor(name_).contiguous()
+                shape = (ctypes.c_size_t * tensor.ndim)(*tensor.shape)
+                LIB_LLAISYS.llaisysQwen2ModelLoadWeight(
+                    self._model, name_.encode("utf-8"), shape, tensor.ndim,
+                    int(_dtype(str(tensor.dtype).replace("torch.", ""))),
+                    ctypes.c_void_p(tensor.data_ptr())
+                )
 
     def generate(
         self,
@@ -28,6 +95,20 @@ def generate(
         temperature: float = 0.8,
     ):
 
-        # TODO: Implement generate function
+        LIB_LLAISYS.llaisysQwen2ModelResetCache(self._model)
+        tokens = [int(x) for x in inputs]
+        steps = 128 if max_new_tokens is None else int(max_new_tokens)
+        for step in range(steps):
+            current = tokens if step == 0 else [tokens[-1]]
+            values = (ctypes.c_int64 * len(current))(*current)
+            token = int(LIB_LLAISYS.llaisysQwen2ModelInfer(self._model, values, len(current)))
+            tokens.append(token)
+            if token == self._meta.end_token:
+                break
+        return tokens
 
-        return []
+    def __del__(self):
+        model = getattr(self, "_model", None)
+        if model:
+            LIB_LLAISYS.llaisysQwen2ModelDestroy(model)
+            self._model = None

python/llaisys/ops.py

@@ -21,7 +21,8 @@ def embedding(out: Tensor, index: Tensor, weight: Tensor):
     @staticmethod
     def linear(out: Tensor, inp: Tensor, weight: Tensor, bias: Tensor):
         LIB_LLAISYS.llaisysLinear(
-            out.lib_tensor(), inp.lib_tensor(), weight.lib_tensor(), bias.lib_tensor()
+            out.lib_tensor(), inp.lib_tensor(), weight.lib_tensor(),
+            bias.lib_tensor() if bias is not None else None
         )
 
     @staticmethod

qwen/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

qwen/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 DeepSeek
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.