talk-llama : sync llama.cpp

ggml-ci

examples/talk-llama/llama-kv-cache-recurrent.h

@@ -32,8 +32,7 @@ public:
     llama_memory_state_ptr init_batch(
             const llama_batch & batch,
             uint32_t n_ubatch,
-            bool embd_pooled,
-            bool logits_all) override;
+            bool embd_all) override;
 
     llama_memory_state_ptr init_full() override;
 
@@ -57,10 +56,6 @@ public:
 
     bool get_can_shift() const override;
 
-    // TODO: temporary methods - they are not really const as they do const_cast<>, fix this
-    int32_t s_copy(int i) const;
-    float   s_mask(int i) const;
-
     // state write/load
 
     void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1) const override;
@@ -73,10 +68,14 @@ public:
     // computed before each graph build
     uint32_t n = 0;
 
+    // first zero-ed state
+    int32_t rs_z = -1;
+
     // TODO: optimize for recurrent state needs
     struct kv_cell {
         llama_pos pos  = -1;
-        int32_t   src  = -1; // used to copy states
+        int32_t   src  = -1; // used to know where states should be copied from
+        int32_t   src0 = -1; // like src, but only used when setting the inputs (allowing to copy once)
         int32_t   tail = -1;
 
         std::set<llama_seq_id> seq_id;
@@ -157,13 +156,13 @@ public:
 
     uint32_t get_n_kv() const;
     uint32_t get_head() const;
+    int32_t  get_rs_z() const;
     uint32_t get_size() const;
 
     ggml_tensor * get_k_l(int32_t il) const;
     ggml_tensor * get_v_l(int32_t il) const;
 
     int32_t s_copy(int i) const;
-    float   s_mask(int i) const;
 
 private:
     const llama_memory_status status;