diff --git a/depth.pfm b/depth.pfm new file mode 100644 index 0000000..1e8e9b3 Binary files /dev/null and b/depth.pfm differ diff --git a/depth.png b/depth.png new file mode 100644 index 0000000..70f6d2d Binary files /dev/null and b/depth.png differ diff --git a/src/dino_backbone.cpp b/src/dino_backbone.cpp index e114f21..bd831ab 100644 --- a/src/dino_backbone.cpp +++ b/src/dino_backbone.cpp @@ -167,8 +167,8 @@ bool DinoBackbone::forward(const std::vector& input_chw, int H, int W, // --- host post-process matching get_intermediate_layers (width == 2*embed) --- // feat = cat([local_x, vit_norm(x)]) over channels, patches 1..256 -> [256,1536] // cam = cat([local_x[token0], x[token0]]) RAW (no norm) -> [1536] - ggml_tensor* nw = ml_.tensor("vit.norm.weight"); - ggml_tensor* nb = ml_.tensor("vit.norm.bias"); + ggml_tensor* nw = ml_.host_tensor("vit.norm.weight"); + ggml_tensor* nb = ml_.host_tensor("vit.norm.bias"); const float* nwp=(const float*)nw->data; const float* nbp=(const float*)nb->data; auto layernorm_host=[&](const float* row)->std::vector{ @@ -566,8 +566,8 @@ bool DinoBackbone::forward_mv_ordered(const std::vector>& vie if (!ok) return false; // Host post-process per view (matches the S=1 path, repeated per view-slice). - ggml_tensor* nw = ml_.tensor("vit.norm.weight"); - ggml_tensor* nb = ml_.tensor("vit.norm.bias"); + ggml_tensor* nw = ml_.host_tensor("vit.norm.weight"); + ggml_tensor* nb = ml_.host_tensor("vit.norm.bias"); const float* nwp=(const float*)nw->data; const float* nbp=(const float*)nb->data; auto layernorm_host=[&](const float* row)->std::vector{ diff --git a/src/model_loader.cpp b/src/model_loader.cpp index 686d557..41d7be3 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -15,16 +15,24 @@ static bool g_gpu_mode = false; void set_gpu_mode(bool on){ g_gpu_mode = on; } bool gpu_mode(){ return g_gpu_mode; } -// Weights read directly on the HOST via ->data during graph build (they feed -// host-computed graph INPUTS, not graph nodes), so they MUST stay in +// Weights read directly on the HOST via ->data during graph build feed +// host-computed graph INPUTS (not graph nodes), so they MUST stay in // host-accessible memory and are never mirrored to the device: // - "vit.pos_embed" : host bicubic interp in DinoBackbone::interp_pos_embed // - "vit.camera_token": host camera-token inject in DinoBackbone::forward* -// - "vit.norm.weight" / "vit.norm.bias": host post-norm in DinoBackbone // (The metric branch aliases m_vit.* -> vit.* so the same names apply.) -static bool is_host_read_tensor(const std::string& name) { - return name == "vit.pos_embed" || name == "vit.camera_token" || - name == "vit.norm.weight" || name == "vit.norm.bias"; +static bool is_host_only_tensor(const std::string& name) { + return name == "vit.pos_embed" || name == "vit.camera_token"; +} + +// Weights that are DUAL-USE: real ggml graph operands in the fused path +// (DinoBackbone::build_feats_graph -> layernorm(x, vit.norm.{w,b})), AND read +// directly on the host in the unfused / multi-view path (layernorm_host). These +// must be mirrored to the device so the graph has a valid backend buffer (else +// the Metal encoder derefs a NULL buffer and segfaults), while their host +// originals are preserved for the host readers via ModelLoader::host_tensor(). +static bool is_host_also_tensor(const std::string& name) { + return name == "vit.norm.weight" || name == "vit.norm.bias"; } static uint32_t kv_u32(gguf_context* g, const char* k, uint32_t d=0){ @@ -192,10 +200,14 @@ bool ModelLoader::offload_weights(Backend& be){ std::unordered_map newmap; newmap.reserve(n); size_t n_dev = 0; for (auto& kv : tensors_) { - if (is_host_read_tensor(kv.first)) { + if (is_host_only_tensor(kv.first)) { newmap.emplace(kv.first, kv.second); // keep host tensor as-is continue; } + // Dual-use weights: preserve the host original for host_tensor() reads, + // then fall through to mirror them onto the device for the graph. + if (is_host_also_tensor(kv.first)) + host_tensors_.emplace(kv.first, kv.second); ggml_tensor* s = kv.second; auto it = src2dev.find(s); if (it != src2dev.end()) { // alias of an already-mirrored tensor @@ -214,8 +226,15 @@ bool ModelLoader::offload_weights(Backend& be){ for (auto& pr : ups) ggml_backend_tensor_set(pr.first, pr.second, 0, ggml_nbytes(pr.first)); tensors_.swap(newmap); // graphs now reference the device-resident weights - DA_LOG("offload_weights: %zu weights -> %s (%zu host-read tensors kept on CPU)", - n_dev, be.device_name().c_str(), (size_t)4); + DA_LOG("offload_weights: %zu weights -> %s (%zu host-only tensors kept on CPU, " + "%zu dual-use weights mirrored + host-preserved)", + n_dev, be.device_name().c_str(), (size_t)2, host_tensors_.size()); return true; } + +ggml_tensor* ModelLoader::host_tensor(const std::string& name) const { + auto it = host_tensors_.find(name); + if (it != host_tensors_.end()) return it->second; // preserved host original + return tensor(name); // CPU path: never offloaded +} } // namespace da diff --git a/src/model_loader.hpp b/src/model_loader.hpp index 45feb73..a72c055 100644 --- a/src/model_loader.hpp +++ b/src/model_loader.hpp @@ -38,12 +38,19 @@ class ModelLoader { bool load(const std::string& path); const Config& config() const { return cfg_; } ggml_tensor* tensor(const std::string& name) const; + // Host-resident copy of a dual-use weight (e.g. vit.norm.{weight,bias}) that is + // BOTH a graph operand (device-resident after offload) AND read directly on the + // host (layernorm_host). Falls back to tensor() when not offloading. Use this at + // every `->data` read site so it stays valid on GPU backends. + ggml_tensor* host_tensor(const std::string& name) const; bool offload_weights(Backend& be); private: Config cfg_; gguf_context* gguf_ = nullptr; ggml_context* ctx_ = nullptr; std::unordered_map tensors_; + // Preserved host originals of dual-use weights mirrored to the device. + std::unordered_map host_tensors_; ggml_context* device_ctx_ = nullptr; ggml_backend_buffer* gpu_buf_ = nullptr; };