llama.cpp 127 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700
  1. /**
  2. * llama.cpp - git 5bf2a2771886ee86137e01dbc7492f78fb392066
  3. *
  4. * MIT License
  5. *
  6. * Copyright (c) 2023 Georgi Gerganov
  7. *
  8. * Permission is hereby granted, free of charge, to any person obtaining a copy
  9. * of this software and associated documentation files (the "Software"), to deal
  10. * in the Software without restriction, including without limitation the rights
  11. * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12. * copies of the Software, and to permit persons to whom the Software is
  13. * furnished to do so, subject to the following conditions:
  14. *
  15. * The above copyright notice and this permission notice shall be included in all
  16. * copies or substantial portions of the Software.
  17. *
  18. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  24. * SOFTWARE.
  25. */
  26. // Defines fileno on msys:
  27. #ifndef _GNU_SOURCE
  28. #define _GNU_SOURCE
  29. #include <cstddef>
  30. #include <cstdint>
  31. #include <cstdio>
  32. #endif
  33. #include "llama-util.h"
  34. #include "llama.h"
  35. #include "ggml.h"
  36. #ifdef GGML_USE_CUBLAS
  37. #include "ggml-cuda.h"
  38. #elif defined(GGML_USE_CLBLAST)
  39. #include "ggml-opencl.h"
  40. #endif
  41. #ifdef GGML_USE_METAL
  42. #include "ggml-metal.h"
  43. #endif
  44. #ifdef GGML_USE_MPI
  45. #include "ggml-mpi.h"
  46. #endif
  47. #ifdef GGML_USE_K_QUANTS
  48. #ifndef QK_K
  49. #ifdef GGML_QKK_64
  50. #define QK_K 64
  51. #else
  52. #define QK_K 256
  53. #endif
  54. #endif
  55. #endif
  56. #include <array>
  57. #include <ctime>
  58. #include <cinttypes>
  59. #include <fstream>
  60. #include <random>
  61. #include <map>
  62. #include <unordered_map>
  63. #include <queue>
  64. #include <cassert>
  65. #include <cstring>
  66. #include <climits>
  67. #include <memory>
  68. #include <algorithm>
  69. #include <initializer_list>
  70. #include <thread>
  71. #include <atomic>
  72. #include <mutex>
  73. #include <sstream>
  74. #include <numeric>
  75. #if defined(_MSC_VER)
  76. #pragma warning(disable: 4244 4267) // possible loss of data
  77. #endif
  78. #define LLAMA_USE_SCRATCH
  79. #define LLAMA_MAX_SCRATCH_BUFFERS 16
  80. // available llama models
  81. enum e_model {
  82. MODEL_UNKNOWN,
  83. MODEL_3B,
  84. MODEL_7B,
  85. MODEL_13B,
  86. MODEL_30B,
  87. MODEL_65B,
  88. };
  89. static const size_t kB = 1024;
  90. static const size_t MB = 1024*1024;
  91. // computed for n_ctx == 2048
  92. // TODO: dynamically determine these sizes
  93. // needs modifications in ggml
  94. typedef void (*offload_func_t)(struct ggml_tensor * tensor);
  95. void llama_nop(struct ggml_tensor * tensor) { // don't offload by default
  96. (void) tensor;
  97. }
  98. //
  99. // ggml helpers
  100. //
  101. static void ggml_graph_compute_helper(std::vector<uint8_t> & buf, ggml_cgraph * graph, int n_threads) {
  102. struct ggml_cplan plan = ggml_graph_plan(graph, n_threads);
  103. if (plan.work_size > 0) {
  104. buf.resize(plan.work_size);
  105. plan.work_data = buf.data();
  106. }
  107. ggml_graph_compute(graph, &plan);
  108. }
  109. //
  110. // memory sizes
  111. //
  112. static const std::map<e_model, size_t> & MEM_REQ_SCRATCH0()
  113. {
  114. static std::map<e_model, size_t> k_sizes = {
  115. { MODEL_3B, 256ull * MB },
  116. { MODEL_7B, 512ull * MB },
  117. { MODEL_13B, 512ull * MB },
  118. { MODEL_30B, 512ull * MB },
  119. { MODEL_65B, 1024ull * MB },
  120. };
  121. return k_sizes;
  122. }
  123. static const std::map<e_model, size_t> & MEM_REQ_SCRATCH1()
  124. {
  125. static std::map<e_model, size_t> k_sizes = {
  126. { MODEL_3B, 256ull * MB },
  127. { MODEL_7B, 512ull * MB },
  128. { MODEL_13B, 512ull * MB },
  129. { MODEL_30B, 512ull * MB },
  130. { MODEL_65B, 1024ull * MB },
  131. };
  132. return k_sizes;
  133. }
  134. // 2*n_embd*n_ctx*n_layer*sizeof(float16)
  135. static const std::map<e_model, size_t> & MEM_REQ_KV_SELF()
  136. {
  137. static std::map<e_model, size_t> k_sizes = {
  138. { MODEL_3B, 682ull * MB },
  139. { MODEL_7B, 1026ull * MB },
  140. { MODEL_13B, 1608ull * MB },
  141. { MODEL_30B, 3124ull * MB },
  142. { MODEL_65B, 5120ull * MB },
  143. };
  144. return k_sizes;
  145. }
  146. // this is mostly needed for temporary mul_mat buffers to dequantize the data
  147. // not actually needed if BLAS is disabled
  148. static const std::map<e_model, size_t> & MEM_REQ_EVAL()
  149. {
  150. static std::map<e_model, size_t> k_sizes = {
  151. { MODEL_3B, 512ull * MB },
  152. { MODEL_7B, 768ull * MB },
  153. { MODEL_13B, 1024ull * MB },
  154. { MODEL_30B, 1280ull * MB },
  155. { MODEL_65B, 1536ull * MB },
  156. };
  157. return k_sizes;
  158. }
  159. // amount of VRAM needed per batch size to hold temporary results
  160. // the values for 3b and 65b are not derived from testing but instead chosen conservatively
  161. static const std::map<e_model, size_t> & VRAM_REQ_SCRATCH_BASE()
  162. {
  163. static std::map<e_model, size_t> k_sizes = {
  164. { MODEL_3B, 512ull * kB },
  165. { MODEL_7B, 512ull * kB },
  166. { MODEL_13B, 640ull * kB },
  167. { MODEL_30B, 768ull * kB },
  168. { MODEL_65B, 1536ull * kB },
  169. };
  170. return k_sizes;
  171. }
  172. // amount of VRAM needed per batch size and context to hold temporary results
  173. // the values for 3b and 65b are not derived from testing but instead chosen conservatively
  174. static const std::map<e_model, size_t> & VRAM_REQ_SCRATCH_PER_CONTEXT()
  175. {
  176. static std::map<e_model, size_t> k_sizes = {
  177. { MODEL_3B, 128ull },
  178. { MODEL_7B, 128ull },
  179. { MODEL_13B, 160ull },
  180. { MODEL_30B, 208ull },
  181. { MODEL_65B, 416ull },
  182. };
  183. return k_sizes;
  184. }
  185. // default hparams (LLaMA 7B)
  186. struct llama_hparams {
  187. uint32_t n_vocab = 32000;
  188. uint32_t n_ctx = 512; // this is provided as user input?
  189. uint32_t n_embd = 4096;
  190. uint32_t n_mult = 256;
  191. uint32_t n_head = 32;
  192. uint32_t n_layer = 32;
  193. uint32_t n_rot = 64;
  194. enum llama_ftype ftype = LLAMA_FTYPE_MOSTLY_F16;
  195. bool operator!=(const llama_hparams & other) const {
  196. return static_cast<bool>(memcmp(this, &other, sizeof(llama_hparams)));
  197. }
  198. };
  199. struct llama_layer {
  200. // normalization
  201. struct ggml_tensor * attention_norm;
  202. // attention
  203. struct ggml_tensor * wq;
  204. struct ggml_tensor * wk;
  205. struct ggml_tensor * wv;
  206. struct ggml_tensor * wo;
  207. // normalization
  208. struct ggml_tensor * ffn_norm;
  209. // ff
  210. struct ggml_tensor * w1;
  211. struct ggml_tensor * w2;
  212. struct ggml_tensor * w3;
  213. };
  214. struct llama_kv_cache {
  215. struct ggml_tensor * k = NULL;
  216. struct ggml_tensor * v = NULL;
  217. struct ggml_context * ctx = NULL;
  218. llama_ctx_buffer buf;
  219. int n; // number of tokens currently in the cache
  220. ~llama_kv_cache() {
  221. if (ctx) {
  222. ggml_free(ctx);
  223. }
  224. #ifdef GGML_USE_CUBLAS
  225. ggml_cuda_free_data(k);
  226. ggml_cuda_free_data(v);
  227. #endif // GGML_USE_CUBLAS
  228. }
  229. };
  230. struct llama_vocab {
  231. using id = int32_t;
  232. using token = std::string;
  233. struct token_score {
  234. token tok;
  235. float score;
  236. };
  237. std::unordered_map<token, id> token_to_id;
  238. std::vector<token_score> id_to_token;
  239. };
  240. struct llama_model {
  241. e_model type = MODEL_UNKNOWN;
  242. llama_hparams hparams;
  243. struct ggml_tensor * tok_embeddings;
  244. struct ggml_tensor * norm;
  245. struct ggml_tensor * output;
  246. std::vector<llama_layer> layers;
  247. int n_gpu_layers;
  248. // context
  249. struct ggml_context * ctx = NULL;
  250. // the model memory buffer
  251. llama_ctx_buffer buf;
  252. // model memory mapped file
  253. std::unique_ptr<llama_mmap> mapping;
  254. // objects representing data potentially being locked in memory
  255. llama_mlock mlock_buf;
  256. llama_mlock mlock_mmap;
  257. // for quantize-stats only
  258. std::vector<std::pair<std::string, struct ggml_tensor *>> tensors_by_name;
  259. int64_t t_load_us = 0;
  260. int64_t t_start_us = 0;
  261. llama_vocab vocab;
  262. ~llama_model() {
  263. if (ctx) {
  264. ggml_free(ctx);
  265. }
  266. #ifdef GGML_USE_CUBLAS
  267. for (size_t i = 0; i < tensors_by_name.size(); ++i) {
  268. ggml_cuda_free_data(tensors_by_name[i].second);
  269. }
  270. ggml_cuda_free_scratch();
  271. #elif defined(GGML_USE_CLBLAST)
  272. for (size_t i = 0; i < tensors_by_name.size(); ++i) {
  273. ggml_cl_free_data(tensors_by_name[i].second);
  274. }
  275. #endif
  276. }
  277. };
  278. struct llama_context {
  279. llama_context(const llama_model & model, const llama_vocab & vocab) : model(model), vocab(vocab), t_load_us(model.t_load_us), t_start_us(model.t_start_us) {}
  280. #ifdef GGML_USE_METAL
  281. ~llama_context() {
  282. if (ctx_metal) {
  283. ggml_metal_free(ctx_metal);
  284. }
  285. }
  286. #endif
  287. std::mt19937 rng;
  288. bool has_evaluated_once = false;
  289. int64_t t_sample_us = 0;
  290. int64_t t_eval_us = 0;
  291. int64_t t_p_eval_us = 0;
  292. int32_t n_sample = 0; // number of tokens sampled
  293. int32_t n_eval = 0; // number of eval calls
  294. int32_t n_p_eval = 0; // number of tokens in eval calls for the prompt (with batch size > 1)
  295. const llama_model & model;
  296. const llama_vocab & vocab;
  297. bool model_owner = false;
  298. int64_t t_load_us;
  299. int64_t t_start_us;
  300. // key + value cache for the self attention
  301. struct llama_kv_cache kv_self;
  302. size_t mem_per_token = 0;
  303. // decode output (2-dimensional array: [n_tokens][n_vocab])
  304. std::vector<float> logits;
  305. bool logits_all = false;
  306. // input embedding (1-dimensional array: [n_embd])
  307. std::vector<float> embedding;
  308. // reusable buffer for `struct ggml_graph_plan.work_data`
  309. std::vector<uint8_t> work_buffer;
  310. // memory buffers used to evaluate the model
  311. // TODO: move in llama_state
  312. llama_ctx_buffer buf_compute;
  313. llama_ctx_buffer buf_scratch[LLAMA_MAX_SCRATCH_BUFFERS];
  314. #ifdef GGML_USE_METAL
  315. ggml_metal_context * ctx_metal = NULL;
  316. #endif
  317. #ifdef GGML_USE_MPI
  318. ggml_mpi_context * ctx_mpi = NULL;
  319. #endif
  320. int buf_last = 0;
  321. size_t buf_max_size[LLAMA_MAX_SCRATCH_BUFFERS] = { 0 };
  322. void use_buf(struct ggml_context * ctx, int i) {
  323. #if defined(LLAMA_USE_SCRATCH)
  324. size_t last_size = 0;
  325. if (i == -1) {
  326. last_size = ggml_set_scratch(ctx, { 0, 0, nullptr, });
  327. } else {
  328. auto & buf = buf_scratch[i];
  329. last_size = ggml_set_scratch(ctx, { 0, buf.size, buf.addr, });
  330. }
  331. if (buf_last >= 0) {
  332. buf_max_size[buf_last] = std::max(buf_max_size[buf_last], last_size);
  333. }
  334. buf_last = i;
  335. #else
  336. (void) i;
  337. (void) ctx;
  338. #endif
  339. }
  340. size_t get_buf_max_mem(int i) const {
  341. #if defined(LLAMA_USE_SCRATCH)
  342. return buf_max_size[i];
  343. #else
  344. (void) i;
  345. return 0;
  346. #endif
  347. }
  348. };
  349. template <typename T>
  350. static T checked_mul(T a, T b) {
  351. T ret = a * b;
  352. if (a != 0 && ret / a != b) {
  353. throw std::runtime_error(format("overflow multiplying %llu * %llu",
  354. (unsigned long long) a, (unsigned long long) b));
  355. }
  356. return ret;
  357. }
  358. static size_t checked_div(size_t a, size_t b) {
  359. if (b == 0 || a % b != 0) {
  360. throw std::runtime_error(format("error dividing %zu / %zu", a, b));
  361. }
  362. return a / b;
  363. }
  364. static std::string llama_format_tensor_shape(const std::vector<uint32_t> & ne) {
  365. char buf[256];
  366. snprintf(buf, sizeof(buf), "%5u", ne.at(0));
  367. for (size_t i = 1; i < ne.size(); i++) {
  368. snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), " x %5u", ne.at(i));
  369. }
  370. return buf;
  371. }
  372. static size_t llama_calc_tensor_size(const std::vector<uint32_t> & ne, enum ggml_type type) {
  373. size_t size = ggml_type_size(type);
  374. for (uint32_t dim : ne) {
  375. size = checked_mul<size_t>(size, dim);
  376. }
  377. return size / ggml_blck_size(type);
  378. }
  379. struct llama_load_tensor {
  380. std::string name;
  381. enum ggml_type type = GGML_TYPE_F32;
  382. std::vector<uint32_t> ne;
  383. size_t file_off;
  384. size_t size;
  385. struct ggml_tensor * ggml_tensor = NULL;
  386. uint8_t * data;
  387. };
  388. struct llama_load_tensors_map {
  389. // tensors is kept in a separate vector to preserve file order
  390. std::vector<llama_load_tensor> tensors;
  391. std::unordered_map<std::string, size_t> name_to_idx;
  392. };
  393. enum llama_file_version {
  394. LLAMA_FILE_VERSION_GGML,
  395. LLAMA_FILE_VERSION_GGMF_V1, // added version field and scores in vocab
  396. LLAMA_FILE_VERSION_GGJT_V1, // added padding
  397. LLAMA_FILE_VERSION_GGJT_V2, // changed quantization format
  398. LLAMA_FILE_VERSION_GGJT_V3, // changed Q4 and Q8 quantization format
  399. };
  400. struct llama_file_loader {
  401. llama_file file;
  402. llama_file_version file_version;
  403. llama_hparams hparams;
  404. llama_vocab vocab;
  405. llama_file_loader(const char * fname, llama_load_tensors_map & tensors_map)
  406. : file(fname, "rb") {
  407. fprintf(stderr, "llama.cpp: loading model from %s\n", fname);
  408. read_magic();
  409. read_hparams();
  410. read_vocab();
  411. read_tensor_metadata(tensors_map);
  412. }
  413. void read_magic() {
  414. uint32_t magic = file.read_u32();
  415. if (magic == LLAMA_FILE_MAGIC_GGML) {
  416. file_version = LLAMA_FILE_VERSION_GGML;
  417. return;
  418. }
  419. uint32_t version = file.read_u32();
  420. switch (magic) {
  421. case LLAMA_FILE_MAGIC_GGMF:
  422. switch (version) {
  423. case 1: file_version = LLAMA_FILE_VERSION_GGMF_V1; return;
  424. }
  425. break;
  426. case LLAMA_FILE_MAGIC_GGJT:
  427. switch (version) {
  428. case 1: file_version = LLAMA_FILE_VERSION_GGJT_V1; return;
  429. case 2: file_version = LLAMA_FILE_VERSION_GGJT_V2; return;
  430. case 3: file_version = LLAMA_FILE_VERSION_GGJT_V3; return;
  431. }
  432. }
  433. throw std::runtime_error(format("unknown (magic, version) combination: %08x, %08x; is this really a GGML file?",
  434. magic, version));
  435. }
  436. void read_hparams() {
  437. hparams.n_vocab = file.read_u32();
  438. hparams.n_embd = file.read_u32();
  439. hparams.n_mult = file.read_u32();
  440. hparams.n_head = file.read_u32();
  441. hparams.n_layer = file.read_u32();
  442. hparams.n_rot = file.read_u32();
  443. hparams.ftype = (enum llama_ftype) file.read_u32();
  444. }
  445. void read_vocab() {
  446. vocab.id_to_token.resize(hparams.n_vocab);
  447. for (uint32_t i = 0; i < hparams.n_vocab; i++) {
  448. uint32_t len = file.read_u32();
  449. std::string word = file.read_string(len);
  450. float score = 0.0f;
  451. file.read_raw(&score, sizeof(score));
  452. vocab.token_to_id[word] = i;
  453. auto & tok_score = vocab.id_to_token[i];
  454. tok_score.tok = std::move(word);
  455. tok_score.score = score;
  456. }
  457. }
  458. void read_tensor_metadata(llama_load_tensors_map & tensors_map) {
  459. while (file.tell() < file.size) {
  460. llama_load_tensor tensor;
  461. uint32_t n_dims = file.read_u32();
  462. uint32_t name_len = file.read_u32();
  463. tensor.type = (enum ggml_type) file.read_u32();
  464. tensor.ne.resize(n_dims);
  465. file.read_raw(tensor.ne.data(), sizeof(tensor.ne[0]) * n_dims);
  466. std::string name = file.read_string(name_len);
  467. if (n_dims < 1 || n_dims > 2) {
  468. throw std::runtime_error(format("llama.cpp: tensor '%s' should not be %u-dimensional", name.c_str(), n_dims));
  469. }
  470. switch (tensor.type) {
  471. case GGML_TYPE_F32:
  472. case GGML_TYPE_F16:
  473. case GGML_TYPE_Q4_0:
  474. case GGML_TYPE_Q4_1:
  475. case GGML_TYPE_Q5_0:
  476. case GGML_TYPE_Q5_1:
  477. case GGML_TYPE_Q8_0:
  478. case GGML_TYPE_Q2_K:
  479. case GGML_TYPE_Q3_K:
  480. case GGML_TYPE_Q4_K:
  481. case GGML_TYPE_Q5_K:
  482. case GGML_TYPE_Q6_K:
  483. break;
  484. default: {
  485. throw std::runtime_error(format("unrecognized tensor type %u\n", tensor.type));
  486. }
  487. }
  488. // skip to the next multiple of 32 bytes
  489. file.seek(-static_cast<ptrdiff_t>(file.tell()) & 31, SEEK_CUR);
  490. tensor.file_off = file.tell();
  491. tensor.name = name;
  492. tensor.size = llama_calc_tensor_size(tensor.ne, tensor.type);
  493. file.seek(tensor.size, SEEK_CUR);
  494. tensors_map.tensors.push_back(tensor);
  495. tensors_map.name_to_idx[name] = tensors_map.tensors.size() - 1;
  496. }
  497. }
  498. };
  499. struct llama_file_saver {
  500. llama_file file;
  501. llama_file_loader * any_file_loader;
  502. llama_file_saver(const char * fname, llama_file_loader * any_file_loader, enum llama_ftype new_ftype)
  503. : file(fname, "wb"), any_file_loader(any_file_loader) {
  504. fprintf(stderr, "llama.cpp: saving model to %s\n", fname);
  505. write_magic();
  506. write_hparams(new_ftype);
  507. write_vocab();
  508. }
  509. void write_magic() {
  510. file.write_u32(LLAMA_FILE_MAGIC); // magic
  511. file.write_u32(LLAMA_FILE_VERSION); // version
  512. }
  513. void write_hparams(enum llama_ftype new_ftype) {
  514. const llama_hparams & hparams = any_file_loader->hparams;
  515. file.write_u32(hparams.n_vocab);
  516. file.write_u32(hparams.n_embd);
  517. file.write_u32(hparams.n_mult);
  518. file.write_u32(hparams.n_head);
  519. file.write_u32(hparams.n_layer);
  520. file.write_u32(hparams.n_rot);
  521. file.write_u32(new_ftype);
  522. }
  523. void write_vocab() {
  524. if (any_file_loader->file_version == LLAMA_FILE_VERSION_GGML) {
  525. fprintf(stderr, "llama.cpp: WARNING: input is an old file that doesn't have scores; will add dummy scores\n");
  526. }
  527. uint32_t n_vocab = any_file_loader->hparams.n_vocab;
  528. for (uint32_t i = 0; i < n_vocab; i++) {
  529. const auto & token_score = any_file_loader->vocab.id_to_token.at(i);
  530. file.write_u32((uint32_t) token_score.tok.size());
  531. file.write_raw(token_score.tok.data(), token_score.tok.size());
  532. file.write_raw(&token_score.score, sizeof(token_score.score));
  533. }
  534. }
  535. void write_tensor(llama_load_tensor & tensor, enum ggml_type new_type, const void * new_data, size_t new_size) {
  536. switch (new_type) {
  537. case GGML_TYPE_F32:
  538. case GGML_TYPE_F16:
  539. case GGML_TYPE_Q4_0:
  540. case GGML_TYPE_Q4_1:
  541. case GGML_TYPE_Q5_0:
  542. case GGML_TYPE_Q5_1:
  543. case GGML_TYPE_Q8_0:
  544. case GGML_TYPE_Q2_K:
  545. case GGML_TYPE_Q3_K:
  546. case GGML_TYPE_Q4_K:
  547. case GGML_TYPE_Q5_K:
  548. case GGML_TYPE_Q6_K:
  549. break;
  550. default: LLAMA_ASSERT(false);
  551. }
  552. file.write_u32((uint32_t) tensor.ne.size());
  553. file.write_u32((uint32_t) tensor.name.size());
  554. file.write_u32(new_type);
  555. file.write_raw(tensor.ne.data(), sizeof(tensor.ne[0]) * tensor.ne.size());
  556. file.write_raw(tensor.name.data(), tensor.name.size());
  557. file.seek(-static_cast<ptrdiff_t>(file.tell()) & 31, SEEK_CUR);
  558. LLAMA_ASSERT(new_size == llama_calc_tensor_size(tensor.ne, new_type));
  559. file.write_raw(new_data, new_size);
  560. }
  561. };
  562. struct llama_model_loader {
  563. std::unique_ptr<llama_file_loader> file_loader;
  564. llama_load_tensors_map tensors_map;
  565. bool use_mmap;
  566. size_t num_ggml_tensors_created = 0;
  567. struct ggml_context * ggml_ctx = NULL;
  568. std::unique_ptr<llama_mmap> mapping;
  569. llama_model_loader(const std::string & fname_base, bool use_mmap) {
  570. file_loader = std::unique_ptr<llama_file_loader>(new llama_file_loader(fname_base.c_str(), tensors_map));
  571. if (!llama_mmap::SUPPORTED) {
  572. use_mmap = false;
  573. }
  574. this->use_mmap = use_mmap;
  575. }
  576. void calc_sizes(size_t * ctx_size_p, size_t * mmapped_size_p) const {
  577. *ctx_size_p = *mmapped_size_p = 0;
  578. for (const llama_load_tensor & lt : tensors_map.tensors) {
  579. *ctx_size_p += sizeof(struct ggml_tensor) + GGML_OBJECT_SIZE;
  580. *(use_mmap ? mmapped_size_p : ctx_size_p) += lt.size;
  581. }
  582. }
  583. struct ggml_tensor * get_tensor(const std::string & name, const std::vector<uint32_t> & ne, ggml_backend backend) {
  584. auto it = tensors_map.name_to_idx.find(name);
  585. if (it == tensors_map.name_to_idx.end()) {
  586. throw std::runtime_error(std::runtime_error(format("llama.cpp: tensor '%s' is missing from model", name.c_str())));
  587. }
  588. llama_load_tensor & lt = tensors_map.tensors.at(it->second);
  589. if (lt.ne != ne) {
  590. throw std::runtime_error(format("llama.cpp: tensor '%s' has wrong shape; expected %s, got %s",
  591. name.c_str(), llama_format_tensor_shape(ne).c_str(), llama_format_tensor_shape(lt.ne).c_str()));
  592. }
  593. return get_tensor_for(lt, backend);
  594. }
  595. struct ggml_tensor * get_tensor_for(llama_load_tensor & lt, ggml_backend backend) {
  596. struct ggml_tensor * tensor;
  597. if (backend != GGML_BACKEND_CPU) {
  598. ggml_set_no_alloc(ggml_ctx, true);
  599. }
  600. if (lt.ne.size() == 2) {
  601. tensor = ggml_new_tensor_2d(ggml_ctx, lt.type, lt.ne.at(0), lt.ne.at(1));
  602. } else {
  603. LLAMA_ASSERT(lt.ne.size() == 1);
  604. tensor = ggml_new_tensor_1d(ggml_ctx, lt.type, lt.ne.at(0));
  605. }
  606. ggml_set_name(tensor, lt.name.c_str());
  607. LLAMA_ASSERT(lt.ggml_tensor == NULL); // if this fails, we called get_tensor twice on the same tensor
  608. if (backend != GGML_BACKEND_CPU) {
  609. ggml_set_no_alloc(ggml_ctx, use_mmap);
  610. }
  611. tensor->backend = backend;
  612. lt.ggml_tensor = tensor;
  613. num_ggml_tensors_created++;
  614. return tensor;
  615. }
  616. void done_getting_tensors() const {
  617. if (num_ggml_tensors_created != tensors_map.tensors.size()) {
  618. throw std::runtime_error(std::string("llama.cpp: file contained more tensors than expected"));
  619. }
  620. }
  621. void load_all_data(llama_progress_callback progress_callback, void * progress_callback_user_data, llama_mlock * lmlock) {
  622. size_t data_size = 0;
  623. size_t prefetch_size = 0;
  624. size_t lock_size = 0;
  625. for (const llama_load_tensor & lt : tensors_map.tensors) {
  626. data_size += lt.size;
  627. if (lt.ggml_tensor->backend == GGML_BACKEND_CPU) {
  628. prefetch_size += lt.size;
  629. }
  630. }
  631. if (use_mmap) {
  632. mapping.reset(new llama_mmap(&file_loader->file, prefetch_size, ggml_is_numa()));
  633. if (lmlock) {
  634. lmlock->init(mapping->addr);
  635. }
  636. }
  637. size_t done_size = 0;
  638. for (llama_load_tensor & lt : tensors_map.tensors) {
  639. if (progress_callback) {
  640. progress_callback((float) done_size / data_size, progress_callback_user_data);
  641. }
  642. LLAMA_ASSERT(lt.ggml_tensor); // unused tensors should have been caught by load_data already
  643. lt.data = (uint8_t *) lt.ggml_tensor->data;
  644. // allocate temp buffer if not using mmap
  645. if (!use_mmap && lt.data == NULL) {
  646. GGML_ASSERT(lt.ggml_tensor->backend != GGML_BACKEND_CPU);
  647. lt.data = (uint8_t*)malloc(ggml_nbytes(lt.ggml_tensor));
  648. }
  649. load_data_for(lt);
  650. switch(lt.ggml_tensor->backend) {
  651. case GGML_BACKEND_CPU:
  652. lt.ggml_tensor->data = lt.data;
  653. if (use_mmap && lmlock) {
  654. lock_size += lt.size;
  655. lmlock->grow_to(lock_size);
  656. }
  657. break;
  658. #if defined(GGML_USE_CUBLAS)
  659. case GGML_BACKEND_GPU:
  660. case GGML_BACKEND_GPU_SPLIT:
  661. ggml_cuda_transform_tensor(lt.data, lt.ggml_tensor);
  662. if (!use_mmap) {
  663. free(lt.data);
  664. }
  665. break;
  666. #elif defined(GGML_USE_CLBLAST)
  667. case GGML_BACKEND_GPU:
  668. ggml_cl_transform_tensor(lt.data, lt.ggml_tensor);
  669. if (!use_mmap) {
  670. free(lt.data);
  671. }
  672. break;
  673. #endif
  674. default:
  675. continue;
  676. }
  677. done_size += lt.size;
  678. }
  679. }
  680. void load_data_for(llama_load_tensor & lt) {
  681. if (use_mmap) {
  682. lt.data = (uint8_t *) mapping->addr + lt.file_off;
  683. } else {
  684. llama_file & file = file_loader->file;
  685. file.seek(lt.file_off, SEEK_SET);
  686. file.read_raw(lt.data, lt.size);
  687. }
  688. if (0) {
  689. print_checksum(lt);
  690. }
  691. }
  692. static void print_checksum(llama_load_tensor & lt) {
  693. uint32_t sum = 0;
  694. for (size_t i = 0; i < lt.size; i++) {
  695. uint8_t byte = lt.data[i];
  696. sum = byte + (sum << 6) + (sum << 16) - sum; // sdbm hash
  697. }
  698. fprintf(stderr, "%s checksum: %#08x (%s, size %zu)\n", lt.name.c_str(), sum,
  699. llama_format_tensor_shape(lt.ne).c_str(), lt.size);
  700. }
  701. };
  702. //
  703. // kv cache
  704. //
  705. static bool kv_cache_init(
  706. const struct llama_hparams & hparams,
  707. struct llama_kv_cache & cache,
  708. ggml_type wtype,
  709. int n_ctx,
  710. int n_gpu_layers) {
  711. const int n_embd = hparams.n_embd;
  712. const int n_layer = hparams.n_layer;
  713. const int64_t n_mem = n_layer*n_ctx;
  714. const int64_t n_elements = n_embd*n_mem;
  715. cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2u*MB);
  716. cache.n = 0;
  717. struct ggml_init_params params;
  718. params.mem_size = cache.buf.size;
  719. params.mem_buffer = cache.buf.addr;
  720. params.no_alloc = false;
  721. cache.ctx = ggml_init(params);
  722. if (!cache.ctx) {
  723. fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__);
  724. return false;
  725. }
  726. cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
  727. cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements);
  728. ggml_set_name(cache.k, "cache_k");
  729. ggml_set_name(cache.v, "cache_v");
  730. (void) n_gpu_layers;
  731. #ifdef GGML_USE_CUBLAS
  732. if (n_gpu_layers > n_layer + 1) {
  733. ggml_cuda_assign_buffers_no_scratch(cache.v);
  734. }
  735. if (n_gpu_layers > n_layer + 2) {
  736. ggml_cuda_assign_buffers_no_scratch(cache.k);
  737. }
  738. #endif // GGML_USE_CUBLAS
  739. return true;
  740. }
  741. struct llama_context_params llama_context_default_params() {
  742. struct llama_context_params result = {
  743. /*.seed =*/ LLAMA_DEFAULT_SEED,
  744. /*.n_ctx =*/ 512,
  745. /*.n_batch =*/ 512,
  746. /*.gpu_layers =*/ 0,
  747. /*.main_gpu =*/ 0,
  748. /*.tensor_split =*/ {0},
  749. /*.progress_callback =*/ nullptr,
  750. /*.progress_callback_user_data =*/ nullptr,
  751. /*.low_vram =*/ false,
  752. /*.f16_kv =*/ true,
  753. /*.logits_all =*/ false,
  754. /*.vocab_only =*/ false,
  755. /*.use_mmap =*/ true,
  756. /*.use_mlock =*/ false,
  757. /*.embedding =*/ false,
  758. };
  759. return result;
  760. }
  761. struct llama_model_quantize_params llama_model_quantize_default_params() {
  762. struct llama_model_quantize_params result = {
  763. /*.nthread =*/ 0,
  764. /*.ftype =*/ LLAMA_FTYPE_MOSTLY_Q5_1,
  765. /*.allow_requantize =*/ false,
  766. /*.quantize_output_tensor =*/ true,
  767. };
  768. return result;
  769. }
  770. bool llama_mmap_supported() {
  771. return llama_mmap::SUPPORTED;
  772. }
  773. bool llama_mlock_supported() {
  774. return llama_mlock::SUPPORTED;
  775. }
  776. void llama_backend_init(bool numa) {
  777. ggml_time_init();
  778. // needed to initialize f16 tables
  779. {
  780. struct ggml_init_params params = { 0, NULL, false };
  781. struct ggml_context * ctx = ggml_init(params);
  782. ggml_free(ctx);
  783. }
  784. if (numa) {
  785. ggml_numa_init();
  786. }
  787. #ifdef GGML_USE_MPI
  788. ggml_mpi_backend_init();
  789. #endif
  790. }
  791. void llama_backend_free() {
  792. #ifdef GGML_USE_MPI
  793. ggml_mpi_backend_free();
  794. #endif
  795. }
  796. int64_t llama_time_us() {
  797. return ggml_time_us();
  798. }
  799. //
  800. // model loading
  801. //
  802. static const char *llama_file_version_name(llama_file_version version) {
  803. switch (version) {
  804. case LLAMA_FILE_VERSION_GGML: return "'ggml' (old version with low tokenizer quality and no mmap support)";
  805. case LLAMA_FILE_VERSION_GGMF_V1: return "ggmf v1 (old version with no mmap support)";
  806. case LLAMA_FILE_VERSION_GGJT_V1: return "ggjt v1 (pre #1405)";
  807. case LLAMA_FILE_VERSION_GGJT_V2: return "ggjt v2 (pre #1508)";
  808. case LLAMA_FILE_VERSION_GGJT_V3: return "ggjt v3 (latest)";
  809. }
  810. return "unknown";
  811. }
  812. static const char *llama_ftype_name(enum llama_ftype ftype) {
  813. switch (ftype) {
  814. case LLAMA_FTYPE_ALL_F32: return "all F32";
  815. case LLAMA_FTYPE_MOSTLY_F16: return "mostly F16";
  816. case LLAMA_FTYPE_MOSTLY_Q4_0: return "mostly Q4_0";
  817. case LLAMA_FTYPE_MOSTLY_Q4_1: return "mostly Q4_1";
  818. case LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16:
  819. return "mostly Q4_1, some F16";
  820. case LLAMA_FTYPE_MOSTLY_Q5_0: return "mostly Q5_0";
  821. case LLAMA_FTYPE_MOSTLY_Q5_1: return "mostly Q5_1";
  822. case LLAMA_FTYPE_MOSTLY_Q8_0: return "mostly Q8_0";
  823. // K-quants
  824. case LLAMA_FTYPE_MOSTLY_Q2_K: return "mostly Q2_K";
  825. case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "mostly Q3_K - Small";
  826. case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "mostly Q3_K - Medium";
  827. case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "mostly Q3_K - Large";
  828. case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "mostly Q4_K - Small";
  829. case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "mostly Q4_K - Medium";
  830. case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "mostly Q5_K - Small";
  831. case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "mostly Q5_K - Medium";
  832. case LLAMA_FTYPE_MOSTLY_Q6_K: return "mostly Q6_K";
  833. default: return "unknown, may not work";
  834. }
  835. }
  836. static const char *llama_model_type_name(e_model type) {
  837. switch (type) {
  838. case MODEL_3B: return "3B";
  839. case MODEL_7B: return "7B";
  840. case MODEL_13B: return "13B";
  841. case MODEL_30B: return "30B";
  842. case MODEL_65B: return "65B";
  843. default: LLAMA_ASSERT(false);
  844. }
  845. }
  846. static void llama_model_load_internal(
  847. const std::string & fname,
  848. llama_model & model,
  849. llama_vocab & vocab,
  850. int n_ctx,
  851. int n_batch,
  852. int n_gpu_layers,
  853. int main_gpu,
  854. const float * tensor_split,
  855. bool low_vram,
  856. ggml_type memory_type,
  857. bool use_mmap,
  858. bool use_mlock,
  859. bool vocab_only,
  860. llama_progress_callback progress_callback,
  861. void * progress_callback_user_data) {
  862. model.t_start_us = ggml_time_us();
  863. std::unique_ptr<llama_model_loader> ml(new llama_model_loader(fname, use_mmap));
  864. vocab = std::move(ml->file_loader->vocab);
  865. model.hparams = ml->file_loader->hparams;
  866. model.n_gpu_layers = n_gpu_layers;
  867. llama_file_version file_version = ml->file_loader->file_version;
  868. auto & hparams = model.hparams;
  869. {
  870. switch (hparams.n_layer) {
  871. case 26: model.type = e_model::MODEL_3B; break;
  872. case 32: model.type = e_model::MODEL_7B; break;
  873. case 40: model.type = e_model::MODEL_13B; break;
  874. case 60: model.type = e_model::MODEL_30B; break;
  875. case 80: model.type = e_model::MODEL_65B; break;
  876. default:
  877. {
  878. if (hparams.n_layer < 32) {
  879. model.type = e_model::MODEL_7B;
  880. }
  881. } break;
  882. }
  883. hparams.n_ctx = n_ctx;
  884. }
  885. const uint32_t n_ff = ((2*(4*hparams.n_embd)/3 + hparams.n_mult - 1)/hparams.n_mult)*hparams.n_mult;
  886. {
  887. fprintf(stderr, "%s: format = %s\n", __func__, llama_file_version_name(file_version));
  888. fprintf(stderr, "%s: n_vocab = %u\n", __func__, hparams.n_vocab);
  889. fprintf(stderr, "%s: n_ctx = %u\n", __func__, hparams.n_ctx);
  890. fprintf(stderr, "%s: n_embd = %u\n", __func__, hparams.n_embd);
  891. fprintf(stderr, "%s: n_mult = %u\n", __func__, hparams.n_mult);
  892. fprintf(stderr, "%s: n_head = %u\n", __func__, hparams.n_head);
  893. fprintf(stderr, "%s: n_layer = %u\n", __func__, hparams.n_layer);
  894. fprintf(stderr, "%s: n_rot = %u\n", __func__, hparams.n_rot);
  895. fprintf(stderr, "%s: ftype = %u (%s)\n", __func__, hparams.ftype, llama_ftype_name(hparams.ftype));
  896. fprintf(stderr, "%s: n_ff = %u\n", __func__, n_ff);
  897. fprintf(stderr, "%s: model size = %s\n", __func__, llama_model_type_name(model.type));
  898. }
  899. if (file_version < LLAMA_FILE_VERSION_GGJT_V2) {
  900. if (hparams.ftype != LLAMA_FTYPE_ALL_F32 &&
  901. hparams.ftype != LLAMA_FTYPE_MOSTLY_F16 &&
  902. hparams.ftype != LLAMA_FTYPE_MOSTLY_Q8_0) {
  903. throw std::runtime_error(format("this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1405)"));
  904. }
  905. }
  906. if (file_version < LLAMA_FILE_VERSION_GGJT_V3) {
  907. if (hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_0 ||
  908. hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_1 ||
  909. hparams.ftype == LLAMA_FTYPE_MOSTLY_Q8_0) {
  910. throw std::runtime_error(format("this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1508)"));
  911. }
  912. }
  913. if (vocab_only) {
  914. return;
  915. }
  916. auto & ctx = model.ctx;
  917. size_t ctx_size;
  918. size_t mmapped_size;
  919. ml->calc_sizes(&ctx_size, &mmapped_size);
  920. fprintf(stderr, "%s: ggml ctx size = %7.2f MB\n", __func__, ctx_size/1024.0/1024.0);
  921. // create the ggml context
  922. {
  923. model.buf.resize(ctx_size);
  924. if (use_mlock) {
  925. model.mlock_buf.init(model.buf.addr);
  926. model.mlock_buf.grow_to(model.buf.size);
  927. }
  928. struct ggml_init_params params = {
  929. /*.mem_size =*/ model.buf.size,
  930. /*.mem_buffer =*/ model.buf.addr,
  931. /*.no_alloc =*/ ml->use_mmap,
  932. };
  933. model.ctx = ggml_init(params);
  934. if (!model.ctx) {
  935. throw std::runtime_error(format("ggml_init() failed"));
  936. }
  937. }
  938. (void) main_gpu;
  939. #if defined(GGML_USE_CUBLAS)
  940. fprintf(stderr, "%s: using CUDA for GPU acceleration\n", __func__);
  941. ggml_cuda_set_main_device(main_gpu);
  942. #define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_GPU
  943. #define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_GPU_SPLIT
  944. #elif defined(GGML_USE_CLBLAST)
  945. fprintf(stderr, "%s: using OpenCL for GPU acceleration\n", __func__);
  946. #define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_GPU
  947. #define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_GPU
  948. #else
  949. #define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_CPU
  950. #define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_CPU
  951. #endif
  952. // prepare memory for the weights
  953. size_t vram_weights = 0;
  954. size_t vram_scratch = 0;
  955. {
  956. const uint32_t n_embd = hparams.n_embd;
  957. const uint32_t n_layer = hparams.n_layer;
  958. const uint32_t n_vocab = hparams.n_vocab;
  959. ml->ggml_ctx = ctx;
  960. model.tok_embeddings = ml->get_tensor("tok_embeddings.weight", {n_embd, n_vocab}, GGML_BACKEND_CPU);
  961. // "output" tensor
  962. {
  963. ggml_backend backend_norm;
  964. ggml_backend backend_output;
  965. if (n_gpu_layers > int(n_layer)) { // NOLINT
  966. // norm is not performance relevant on its own but keeping it in VRAM reduces data copying
  967. // on Windows however this is detrimental unless everything is on the GPU
  968. #ifndef _WIN32
  969. backend_norm = low_vram ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD;
  970. #else
  971. backend_norm = low_vram || n_gpu_layers <= (int) n_layer + 2 ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD;
  972. #endif // _WIN32
  973. backend_output = LLAMA_BACKEND_OFFLOAD_SPLIT;
  974. } else {
  975. backend_norm = GGML_BACKEND_CPU;
  976. backend_output = GGML_BACKEND_CPU;
  977. }
  978. model.norm = ml->get_tensor("norm.weight", {n_embd}, backend_norm);
  979. model.output = ml->get_tensor("output.weight", {n_embd, n_vocab}, backend_output);
  980. if (backend_norm == GGML_BACKEND_GPU) {
  981. vram_weights += ggml_nbytes(model.norm);
  982. }
  983. if (backend_output == GGML_BACKEND_GPU_SPLIT) {
  984. vram_weights += ggml_nbytes(model.output);
  985. }
  986. }
  987. const int i_gpu_start = n_layer - n_gpu_layers;
  988. model.layers.resize(n_layer);
  989. for (uint32_t i = 0; i < n_layer; ++i) {
  990. const ggml_backend backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD; // NOLINT
  991. const ggml_backend backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD_SPLIT; // NOLINT
  992. auto & layer = model.layers[i];
  993. std::string layers_i = "layers." + std::to_string(i);
  994. layer.attention_norm = ml->get_tensor(layers_i + ".attention_norm.weight", {n_embd}, backend);
  995. layer.wq = ml->get_tensor(layers_i + ".attention.wq.weight", {n_embd, n_embd}, backend_split);
  996. layer.wk = ml->get_tensor(layers_i + ".attention.wk.weight", {n_embd, n_embd}, backend_split);
  997. layer.wv = ml->get_tensor(layers_i + ".attention.wv.weight", {n_embd, n_embd}, backend_split);
  998. layer.wo = ml->get_tensor(layers_i + ".attention.wo.weight", {n_embd, n_embd}, backend_split);
  999. layer.ffn_norm = ml->get_tensor(layers_i + ".ffn_norm.weight", {n_embd}, backend);
  1000. layer.w1 = ml->get_tensor(layers_i + ".feed_forward.w1.weight", {n_embd, n_ff}, backend_split);
  1001. layer.w2 = ml->get_tensor(layers_i + ".feed_forward.w2.weight", { n_ff, n_embd}, backend_split);
  1002. layer.w3 = ml->get_tensor(layers_i + ".feed_forward.w3.weight", {n_embd, n_ff}, backend_split);
  1003. if (backend == GGML_BACKEND_GPU) {
  1004. vram_weights +=
  1005. ggml_nbytes(layer.attention_norm) + ggml_nbytes(layer.wq) + ggml_nbytes(layer.wk) +
  1006. ggml_nbytes(layer.wv) + ggml_nbytes(layer.wo) + ggml_nbytes(layer.ffn_norm) +
  1007. ggml_nbytes(layer.w1) + ggml_nbytes(layer.w2) + ggml_nbytes(layer.w3);
  1008. }
  1009. }
  1010. }
  1011. ml->done_getting_tensors();
  1012. // print memory requirements
  1013. {
  1014. const size_t scale = memory_type == GGML_TYPE_F32 ? 2 : 1;
  1015. // this is the total memory required to run the inference
  1016. const size_t mem_required =
  1017. ctx_size +
  1018. mmapped_size - vram_weights + // weights in VRAM not in memory
  1019. MEM_REQ_SCRATCH0().at(model.type) +
  1020. MEM_REQ_SCRATCH1().at(model.type) +
  1021. MEM_REQ_EVAL().at (model.type);
  1022. // this is the memory required by one llama_state
  1023. const size_t mem_required_state =
  1024. scale*MEM_REQ_KV_SELF().at(model.type);
  1025. fprintf(stderr, "%s: mem required = %7.2f MB (+ %7.2f MB per state)\n", __func__,
  1026. mem_required / 1024.0 / 1024.0, mem_required_state / 1024.0 / 1024.0);
  1027. (void) vram_scratch;
  1028. (void) n_batch;
  1029. #ifdef GGML_USE_CUBLAS
  1030. if (low_vram) {
  1031. fprintf(stderr, "%s: not allocating a VRAM scratch buffer due to low VRAM option\n", __func__);
  1032. ggml_cuda_set_scratch_size(0); // disable scratch
  1033. } else {
  1034. const size_t vram_scratch_base = VRAM_REQ_SCRATCH_BASE().at(model.type);
  1035. const size_t vram_scratch_per_context = VRAM_REQ_SCRATCH_PER_CONTEXT().at(model.type);
  1036. vram_scratch = n_batch * (vram_scratch_base + n_ctx * vram_scratch_per_context);
  1037. ggml_cuda_set_scratch_size(vram_scratch);
  1038. if (n_gpu_layers > 0) {
  1039. fprintf(stderr, "%s: allocating batch_size x (%zd kB + n_ctx x %zd B) = %zd MB VRAM for the scratch buffer\n",
  1040. __func__, vram_scratch_base / kB, vram_scratch_per_context,
  1041. (vram_scratch + MB - 1) / MB); // round up
  1042. }
  1043. }
  1044. #endif // GGML_USE_CUBLAS
  1045. #if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST)
  1046. const int n_gpu = std::min(n_gpu_layers, int(hparams.n_layer));
  1047. fprintf(stderr, "%s: offloading %d repeating layers to GPU\n", __func__, n_gpu);
  1048. if (n_gpu_layers > (int) hparams.n_layer) {
  1049. fprintf(stderr, "%s: offloading non-repeating layers to GPU\n", __func__);
  1050. }
  1051. size_t vram_kv_cache = 0;
  1052. #ifdef GGML_USE_CUBLAS
  1053. const int max_backend_supported_layers = hparams.n_layer + 3;
  1054. const int max_offloadable_layers = low_vram ? hparams.n_layer + 1 : hparams.n_layer + 3;
  1055. if (n_gpu_layers > (int) hparams.n_layer + 1) {
  1056. if (low_vram) {
  1057. fprintf(stderr, "%s: cannot offload v cache to GPU due to low VRAM option\n", __func__);
  1058. } else {
  1059. fprintf(stderr, "%s: offloading v cache to GPU\n", __func__);
  1060. vram_kv_cache += MEM_REQ_KV_SELF().at(model.type) / 2;
  1061. }
  1062. }
  1063. if (n_gpu_layers > (int) hparams.n_layer + 2) {
  1064. if (low_vram) {
  1065. fprintf(stderr, "%s: cannot offload k cache to GPU due to low VRAM option\n", __func__);
  1066. } else {
  1067. fprintf(stderr, "%s: offloading k cache to GPU\n", __func__);
  1068. vram_kv_cache += MEM_REQ_KV_SELF().at(model.type) / 2;
  1069. }
  1070. }
  1071. #elif defined(GGML_USE_CLBLAST)
  1072. const int max_backend_supported_layers = hparams.n_layer + 1;
  1073. const int max_offloadable_layers = hparams.n_layer + 1;
  1074. #endif // GGML_USE_CUBLAS
  1075. fprintf(stderr, "%s: offloaded %d/%d layers to GPU\n",
  1076. __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers);
  1077. fprintf(stderr, "%s: total VRAM used: %zu MB\n",
  1078. __func__, (vram_weights + vram_scratch + vram_kv_cache + MB - 1) / MB); // round up
  1079. #else
  1080. (void) n_gpu_layers;
  1081. #endif // defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST)
  1082. }
  1083. // populate `tensors_by_name`
  1084. for (llama_load_tensor & lt : ml->tensors_map.tensors) {
  1085. model.tensors_by_name.emplace_back(lt.name, lt.ggml_tensor);
  1086. }
  1087. (void) tensor_split;
  1088. #if defined(GGML_USE_CUBLAS)
  1089. {
  1090. ggml_cuda_set_tensor_split(tensor_split);
  1091. }
  1092. #endif
  1093. ml->load_all_data(progress_callback, progress_callback_user_data, use_mlock ? &model.mlock_mmap : NULL);
  1094. if (progress_callback) {
  1095. progress_callback(1.0f, progress_callback_user_data);
  1096. }
  1097. model.mapping = std::move(ml->mapping);
  1098. // loading time will be recalculate after the first eval, so
  1099. // we take page faults deferred by mmap() into consideration
  1100. model.t_load_us = ggml_time_us() - model.t_start_us;
  1101. }
  1102. static bool llama_model_load(
  1103. const std::string & fname,
  1104. llama_model & model,
  1105. llama_vocab & vocab,
  1106. int n_ctx,
  1107. int n_batch,
  1108. int n_gpu_layers,
  1109. int main_gpu,
  1110. float * tensor_split,
  1111. bool low_vram,
  1112. ggml_type memory_type,
  1113. bool use_mmap,
  1114. bool use_mlock,
  1115. bool vocab_only,
  1116. llama_progress_callback progress_callback,
  1117. void *progress_callback_user_data) {
  1118. try {
  1119. llama_model_load_internal(fname, model, vocab, n_ctx, n_batch, n_gpu_layers, main_gpu, tensor_split, low_vram, memory_type,
  1120. use_mmap, use_mlock, vocab_only, progress_callback, progress_callback_user_data);
  1121. return true;
  1122. } catch (const std::exception & err) {
  1123. fprintf(stderr, "error loading model: %s\n", err.what());
  1124. return false;
  1125. }
  1126. }
  1127. // evaluate the transformer
  1128. //
  1129. // - lctx: llama context
  1130. // - tokens: new batch of tokens to process
  1131. // - embd embeddings input
  1132. // - n_tokens number of tokens
  1133. // - n_past: the context size so far
  1134. // - n_threads: number of threads to use
  1135. //
  1136. static bool llama_eval_internal(
  1137. llama_context & lctx,
  1138. const llama_token * tokens,
  1139. const float * embd,
  1140. int n_tokens,
  1141. int n_past,
  1142. int n_threads,
  1143. const char * cgraph_fname) {
  1144. LLAMA_ASSERT((!tokens && embd) || (tokens && !embd));
  1145. #ifdef GGML_USE_MPI
  1146. ggml_mpi_eval_init(lctx.ctx_mpi, &n_tokens, &n_past, &n_threads);
  1147. #endif
  1148. const int64_t t_start_us = ggml_time_us();
  1149. const int N = n_tokens;
  1150. const auto & model = lctx.model;
  1151. const auto & hparams = model.hparams;
  1152. const auto & kv_self = lctx.kv_self;
  1153. LLAMA_ASSERT(!!kv_self.ctx);
  1154. const int n_embd = hparams.n_embd;
  1155. const int n_layer = hparams.n_layer;
  1156. const int n_ctx = hparams.n_ctx;
  1157. const int n_head = hparams.n_head;
  1158. const int n_vocab = hparams.n_vocab;
  1159. const int n_rot = hparams.n_embd/hparams.n_head;
  1160. const int n_gpu_layers = model.n_gpu_layers;
  1161. auto & mem_per_token = lctx.mem_per_token;
  1162. auto & buf_compute = lctx.buf_compute;
  1163. struct ggml_init_params params = {
  1164. /*.mem_size =*/ buf_compute.size,
  1165. /*.mem_buffer =*/ buf_compute.addr,
  1166. /*.no_alloc =*/ false,
  1167. };
  1168. struct ggml_context * ctx0 = ggml_init(params);
  1169. ggml_cgraph gf = {};
  1170. // for big prompts, if BLAS is enabled, it is better to use only one thread
  1171. // otherwise, the threads are spin-lock waiting for the BLAS calls and are degrading the performance
  1172. n_threads = N >= 32 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas() ? 1 : n_threads;
  1173. struct ggml_tensor * cur;
  1174. struct ggml_tensor * inpL;
  1175. if (tokens) {
  1176. struct ggml_tensor * inp_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
  1177. memcpy(inp_tokens->data, tokens, N*ggml_element_size(inp_tokens));
  1178. ggml_set_name(inp_tokens, "inp_tokens");
  1179. inpL = ggml_get_rows(ctx0, model.tok_embeddings, inp_tokens);
  1180. } else {
  1181. #ifdef GGML_USE_MPI
  1182. GGML_ASSERT(false && "not implemented");
  1183. #endif
  1184. inpL = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N);
  1185. memcpy(inpL->data, embd, N * n_embd * ggml_element_size(inpL));
  1186. }
  1187. const int i_gpu_start = n_layer - n_gpu_layers;
  1188. (void) i_gpu_start;
  1189. // offload functions set the tensor output backend to GPU
  1190. // tensors are GPU-accelerated if any input or the output has been offloaded
  1191. //
  1192. // with the low VRAM option VRAM scratch is disabled in llama_load_model_internal
  1193. // in that case ggml_cuda_assign_buffers has no effect
  1194. offload_func_t offload_func_nr = llama_nop; // nr = non-repeating
  1195. offload_func_t offload_func_kq = llama_nop;
  1196. offload_func_t offload_func_v = llama_nop;
  1197. #ifdef GGML_USE_CUBLAS
  1198. if (n_gpu_layers > n_layer) {
  1199. offload_func_nr = ggml_cuda_assign_buffers;
  1200. }
  1201. if (n_gpu_layers > n_layer + 1) {
  1202. offload_func_v = ggml_cuda_assign_buffers;
  1203. }
  1204. if (n_gpu_layers > n_layer + 2) {
  1205. offload_func_kq = ggml_cuda_assign_buffers;
  1206. }
  1207. #endif // GGML_USE_CUBLAS
  1208. for (int il = 0; il < n_layer; ++il) {
  1209. ggml_format_name(inpL, "layer_inp_%d", il);
  1210. offload_func_t offload_func = llama_nop;
  1211. #ifdef GGML_USE_CUBLAS
  1212. if (il >= i_gpu_start) {
  1213. offload_func = ggml_cuda_assign_buffers;
  1214. }
  1215. #endif // GGML_USE_CUBLAS
  1216. struct ggml_tensor * inpSA = inpL;
  1217. lctx.use_buf(ctx0, 0);
  1218. // norm
  1219. {
  1220. cur = ggml_rms_norm(ctx0, inpL);
  1221. offload_func(cur);
  1222. ggml_set_name(cur, "rms_norm_0");
  1223. // cur = cur*attention_norm(broadcasted)
  1224. cur = ggml_mul(ctx0, cur, model.layers[il].attention_norm);
  1225. offload_func(cur);
  1226. ggml_set_name(cur, "attention_norm_0");
  1227. }
  1228. // self-attention
  1229. {
  1230. // compute Q and K and RoPE them
  1231. struct ggml_tensor * tmpk = ggml_mul_mat(ctx0, model.layers[il].wk, cur);
  1232. offload_func_kq(tmpk);
  1233. ggml_set_name(tmpk, "tmpk");
  1234. struct ggml_tensor * tmpq = ggml_mul_mat(ctx0, model.layers[il].wq, cur);
  1235. offload_func_kq(tmpq);
  1236. ggml_set_name(tmpq, "tmpq");
  1237. struct ggml_tensor * Kcur = ggml_rope_inplace(ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
  1238. offload_func_kq(Kcur);
  1239. ggml_set_name(Kcur, "Kcur");
  1240. struct ggml_tensor * Qcur = ggml_rope_inplace(ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd/n_head, n_head, N), n_past, n_rot, 0, 0);
  1241. offload_func_kq(Qcur);
  1242. ggml_set_name(Qcur, "Qcur");
  1243. // store key and value to memory
  1244. {
  1245. // compute the transposed [N, n_embd] V matrix
  1246. struct ggml_tensor * tmpv = ggml_mul_mat(ctx0, model.layers[il].wv, cur);
  1247. offload_func_v(tmpv);
  1248. ggml_set_name(tmpv, "tmpv");
  1249. struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_reshape_2d(ctx0, tmpv, n_embd, N));
  1250. offload_func_v(Vcur);
  1251. ggml_set_name(Vcur, "Vcur");
  1252. struct ggml_tensor * k = ggml_view_1d(ctx0, kv_self.k, N*n_embd, (ggml_element_size(kv_self.k)*n_embd)*(il*n_ctx + n_past));
  1253. offload_func_kq(k);
  1254. ggml_set_name(k, "k");
  1255. struct ggml_tensor * v = ggml_view_2d(ctx0, kv_self.v, N, n_embd,
  1256. ( n_ctx)*ggml_element_size(kv_self.v),
  1257. (il*n_ctx)*ggml_element_size(kv_self.v)*n_embd + n_past*ggml_element_size(kv_self.v));
  1258. offload_func_v(v);
  1259. ggml_set_name(v, "v");
  1260. // important: storing RoPE-ed version of K in the KV cache!
  1261. ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k));
  1262. ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v));
  1263. }
  1264. struct ggml_tensor * Q =
  1265. ggml_permute(ctx0,
  1266. Qcur,
  1267. 0, 2, 1, 3);
  1268. offload_func_kq(Q);
  1269. ggml_set_name(Q, "Q");
  1270. struct ggml_tensor * K =
  1271. ggml_permute(ctx0,
  1272. ggml_reshape_3d(ctx0,
  1273. ggml_view_1d(ctx0, kv_self.k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(kv_self.k)*n_embd),
  1274. n_embd/n_head, n_head, n_past + N),
  1275. 0, 2, 1, 3);
  1276. offload_func_kq(K);
  1277. ggml_set_name(K, "K");
  1278. // K * Q
  1279. struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
  1280. offload_func_kq(KQ);
  1281. ggml_set_name(KQ, "KQ");
  1282. // KQ_scaled = KQ / sqrt(n_embd/n_head)
  1283. struct ggml_tensor * KQ_scale = ggml_new_f32(ctx0, 1.0f/sqrtf(float(n_embd)/n_head));
  1284. ggml_set_name(KQ_scale, "1/sqrt(n_embd/n_head)");
  1285. // KQ_scaled shape [n_past + N, N, n_head, 1]
  1286. struct ggml_tensor * KQ_scaled = ggml_scale_inplace(ctx0, KQ, KQ_scale);
  1287. offload_func_kq(KQ_scaled);
  1288. ggml_set_name(KQ_scaled, "KQ_scaled");
  1289. // KQ_masked = mask_past(KQ_scaled)
  1290. struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
  1291. offload_func_kq(KQ_masked);
  1292. ggml_set_name(KQ_masked, "KQ_masked");
  1293. // KQ = soft_max(KQ_masked)
  1294. struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
  1295. offload_func_v(KQ_soft_max);
  1296. ggml_set_name(KQ_soft_max, "KQ_soft_max");
  1297. // split cached V into n_head heads
  1298. struct ggml_tensor * V =
  1299. ggml_view_3d(ctx0, kv_self.v,
  1300. n_past + N, n_embd/n_head, n_head,
  1301. n_ctx*ggml_element_size(kv_self.v),
  1302. n_ctx*ggml_element_size(kv_self.v)*n_embd/n_head,
  1303. il*n_ctx*ggml_element_size(kv_self.v)*n_embd);
  1304. offload_func_v(V);
  1305. ggml_set_name(V, "V");
  1306. #if 1
  1307. struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
  1308. offload_func_v(KQV);
  1309. ggml_set_name(KQV, "KQV");
  1310. #else
  1311. // make V contiguous in memory to speed up the matmul, however we waste time on the copy
  1312. // on M1 this is faster for the perplexity computation, but ~5% slower for the single-token generation
  1313. // is there a better way?
  1314. struct ggml_tensor * V_cont = ggml_cpy(ctx0, V, ggml_new_tensor_3d(ctx0, kv_self.v->type, n_past + N, n_embd/n_head, n_head));
  1315. struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_cont, KQ_soft_max);
  1316. #endif
  1317. // KQV_merged = KQV.permute(0, 2, 1, 3)
  1318. struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
  1319. offload_func_v(KQV_merged);
  1320. ggml_set_name(KQV_merged, "KQV_merged");
  1321. // cur = KQV_merged.contiguous().view(n_embd, N)
  1322. cur = ggml_cpy(ctx0,
  1323. KQV_merged,
  1324. ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
  1325. offload_func_v(cur);
  1326. ggml_set_name(cur, "KQV_merged_contiguous");
  1327. // projection (no bias)
  1328. cur = ggml_mul_mat(ctx0,
  1329. model.layers[il].wo,
  1330. cur);
  1331. offload_func(cur);
  1332. ggml_set_name(cur, "result_wo");
  1333. }
  1334. lctx.use_buf(ctx0, 1);
  1335. struct ggml_tensor * inpFF = ggml_add(ctx0, cur, inpSA);
  1336. offload_func(inpFF);
  1337. ggml_set_name(inpFF, "inpFF");
  1338. // feed-forward network
  1339. {
  1340. // norm
  1341. {
  1342. cur = ggml_rms_norm(ctx0, inpFF);
  1343. offload_func(cur);
  1344. ggml_set_name(cur, "rms_norm_1");
  1345. // cur = cur*ffn_norm(broadcasted)
  1346. cur = ggml_mul(ctx0, cur, model.layers[il].ffn_norm);
  1347. offload_func(cur);
  1348. ggml_set_name(cur, "ffn_norm");
  1349. }
  1350. struct ggml_tensor * tmp = ggml_mul_mat(ctx0,
  1351. model.layers[il].w3,
  1352. cur);
  1353. offload_func(tmp);
  1354. ggml_set_name(tmp, "result_w3");
  1355. cur = ggml_mul_mat(ctx0,
  1356. model.layers[il].w1,
  1357. cur);
  1358. offload_func(cur);
  1359. ggml_set_name(cur, "result_w1");
  1360. // SILU activation
  1361. cur = ggml_silu(ctx0, cur);
  1362. offload_func(cur);
  1363. ggml_set_name(cur, "silu");
  1364. cur = ggml_mul(ctx0, cur, tmp);
  1365. offload_func(cur);
  1366. ggml_set_name(cur, "silu_x_result_w3");
  1367. cur = ggml_mul_mat(ctx0,
  1368. model.layers[il].w2,
  1369. cur);
  1370. offload_func(cur);
  1371. ggml_set_name(cur, "result_w2");
  1372. }
  1373. cur = ggml_add(ctx0, cur, inpFF);
  1374. offload_func(cur);
  1375. ggml_set_name(cur, "inpFF_+_result_w2");
  1376. // input for next layer
  1377. inpL = cur;
  1378. }
  1379. lctx.use_buf(ctx0, 0);
  1380. // used at the end to optionally extract the embeddings
  1381. struct ggml_tensor * embeddings = NULL;
  1382. // norm
  1383. {
  1384. cur = ggml_rms_norm(ctx0, inpL);
  1385. offload_func_nr(cur);
  1386. ggml_set_name(cur, "rms_norm_2");
  1387. // cur = cur*norm(broadcasted)
  1388. cur = ggml_mul(ctx0, cur, model.norm);
  1389. // offload_func_nr(cur); // TODO CPU + GPU mirrored backend
  1390. ggml_set_name(cur, "result_norm");
  1391. embeddings = cur;
  1392. }
  1393. // lm_head
  1394. cur = ggml_mul_mat(ctx0, model.output, cur);
  1395. ggml_set_name(cur, "result_output");
  1396. lctx.use_buf(ctx0, -1);
  1397. // logits -> probs
  1398. //cur = ggml_soft_max_inplace(ctx0, cur);
  1399. // run the computation
  1400. ggml_build_forward_expand(&gf, cur);
  1401. #if GGML_USE_MPI
  1402. ggml_mpi_graph_compute_pre(lctx.ctx_mpi, &gf, n_layer);
  1403. #endif
  1404. #ifdef GGML_USE_METAL
  1405. if (lctx.ctx_metal && N == 1) {
  1406. ggml_metal_set_n_cb (lctx.ctx_metal, n_threads);
  1407. ggml_metal_graph_compute(lctx.ctx_metal, &gf);
  1408. ggml_metal_get_tensor (lctx.ctx_metal, cur);
  1409. } else {
  1410. // IMPORTANT:
  1411. // Since we don't have efficient Matrix x Matrix Metal multiplication yet, we fallback to vanilla
  1412. // ggml_graph_compute(). It uses Apple's Accelerate CBLAS API which takes advantage of the ANE or the AMX
  1413. // coprocessor.
  1414. //
  1415. // When we implement Matrix x Matrix Metal multiplication, we can avoid this branch.
  1416. // But for now, we have focused only on Matrix x Vector Metal multiplication.
  1417. //
  1418. // TODO: avoid these syncs via shared memory (ref #1696)
  1419. //
  1420. if (lctx.ctx_metal) {
  1421. // We need to sync the GPU KV cache with the CPU KV cache
  1422. ggml_metal_get_tensor(lctx.ctx_metal, kv_self.k);
  1423. ggml_metal_get_tensor(lctx.ctx_metal, kv_self.v);
  1424. }
  1425. ggml_graph_compute_helper(lctx.work_buffer, &gf, n_threads);
  1426. }
  1427. #else
  1428. ggml_graph_compute_helper(lctx.work_buffer, &gf, n_threads);
  1429. #endif
  1430. #if GGML_USE_MPI
  1431. ggml_mpi_graph_compute_post(lctx.ctx_mpi, &gf, n_layer);
  1432. #endif
  1433. // update kv token count
  1434. lctx.kv_self.n = n_past + N;
  1435. struct ggml_tensor * res = gf.nodes[gf.n_nodes - 1];
  1436. if (cgraph_fname) {
  1437. ggml_graph_export(&gf, cgraph_fname);
  1438. }
  1439. #ifdef GGML_PERF
  1440. // print timing information per ggml operation (for debugging purposes)
  1441. // requires GGML_PERF to be defined
  1442. ggml_graph_print(&gf);
  1443. #endif
  1444. // plot the computation graph in dot format (for debugging purposes)
  1445. //if (n_past%100 == 0) {
  1446. // ggml_graph_dump_dot(&gf, NULL, "llama.dot");
  1447. //}
  1448. // extract logits
  1449. {
  1450. auto & logits_out = lctx.logits;
  1451. if (lctx.logits_all) {
  1452. logits_out.resize(n_vocab * N);
  1453. memcpy(logits_out.data(), (float *) ggml_get_data(res), sizeof(float)*n_vocab*N);
  1454. } else {
  1455. // return result for just the last token
  1456. logits_out.resize(n_vocab);
  1457. memcpy(logits_out.data(), (float *) ggml_get_data(res) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
  1458. }
  1459. }
  1460. // extract embeddings
  1461. if (!lctx.embedding.empty()) {
  1462. auto & embedding_out = lctx.embedding;
  1463. embedding_out.resize(n_embd);
  1464. memcpy(embedding_out.data(), (float *) ggml_get_data(embeddings) + (n_embd*(N - 1)), sizeof(float)*n_embd);
  1465. }
  1466. if (mem_per_token == 0) {
  1467. mem_per_token = ggml_used_mem(ctx0)/N;
  1468. }
  1469. #if 0
  1470. printf("\n%s: used_mem = %.3f MB, scratch -- %.3f MB %.3f MB\n", __func__,
  1471. ggml_used_mem(ctx0)/1024.0/1024.0,
  1472. lctx.get_buf_max_mem(0)/1024.0/1024.0,
  1473. lctx.get_buf_max_mem(1)/1024.0/1024.0);
  1474. #endif
  1475. ggml_free(ctx0);
  1476. // measure the performance only for the single-token evals
  1477. if (N == 1) {
  1478. lctx.t_eval_us += ggml_time_us() - t_start_us;
  1479. lctx.n_eval++;
  1480. }
  1481. else if (N > 1) {
  1482. lctx.t_p_eval_us += ggml_time_us() - t_start_us;
  1483. lctx.n_p_eval += N;
  1484. }
  1485. return true;
  1486. }
  1487. //
  1488. // tokenizer
  1489. //
  1490. static size_t utf8_len(char src) {
  1491. const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };
  1492. uint8_t highbits = static_cast<uint8_t>(src) >> 4;
  1493. return lookup[highbits];
  1494. }
  1495. struct llama_sp_symbol {
  1496. using index = int;
  1497. index prev;
  1498. index next;
  1499. const char * text;
  1500. size_t n;
  1501. };
  1502. static_assert(std::is_trivially_copyable<llama_sp_symbol>::value, "llama_sp_symbol is not trivially copyable");
  1503. struct llama_sp_bigram {
  1504. struct comparator {
  1505. bool operator()(llama_sp_bigram & l, llama_sp_bigram & r) {
  1506. return (l.score < r.score) || (l.score == r.score && l.left > r.left);
  1507. }
  1508. };
  1509. using queue_storage = std::vector<llama_sp_bigram>;
  1510. using queue = std::priority_queue<llama_sp_bigram, queue_storage, comparator>;
  1511. llama_sp_symbol::index left;
  1512. llama_sp_symbol::index right;
  1513. float score;
  1514. size_t size;
  1515. };
  1516. // original implementation:
  1517. // https://github.com/ggerganov/llama.cpp/commit/074bea2eb1f1349a0118239c4152914aecaa1be4
  1518. struct llama_tokenizer {
  1519. llama_tokenizer(const llama_vocab & vocab): vocab_(vocab) {}
  1520. void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
  1521. // split string into utf8 chars
  1522. int index = 0;
  1523. size_t offs = 0;
  1524. while (offs < text.size()) {
  1525. llama_sp_symbol sym;
  1526. size_t char_len = std::min(text.size() - offs, utf8_len(text[offs]));
  1527. sym.text = text.c_str() + offs;
  1528. sym.n = char_len;
  1529. offs += char_len;
  1530. sym.prev = index - 1;
  1531. sym.next = offs == text.size() ? -1 : index + 1;
  1532. index++;
  1533. symbols_.emplace_back(sym);
  1534. }
  1535. // seed the work queue with all possible 2-character tokens.
  1536. for (size_t i = 1; i < symbols_.size(); ++i) {
  1537. try_add_bigram(i - 1, i);
  1538. }
  1539. // keep substituting the highest frequency pairs for as long as we can.
  1540. while (!work_queue_.empty()) {
  1541. auto bigram = work_queue_.top();
  1542. work_queue_.pop();
  1543. auto & left_sym = symbols_[bigram.left];
  1544. auto & right_sym = symbols_[bigram.right];
  1545. // if one of the symbols already got merged, skip it.
  1546. if (left_sym.n == 0 || right_sym.n == 0 ||
  1547. left_sym.n + right_sym.n != bigram.size) {
  1548. continue;
  1549. }
  1550. // merge the right sym into the left one
  1551. left_sym.n += right_sym.n;
  1552. right_sym.n = 0;
  1553. //printf("left = '%*s' size = %zu\n", (int) left_sym.n, left_sym.text, bigram.size);
  1554. // remove the right sym from the chain
  1555. left_sym.next = right_sym.next;
  1556. if (right_sym.next >= 0) {
  1557. symbols_[right_sym.next].prev = bigram.left;
  1558. }
  1559. // find more substitutions
  1560. try_add_bigram(left_sym.prev, bigram.left);
  1561. try_add_bigram(bigram.left, left_sym.next);
  1562. }
  1563. for (int i = 0; i != -1; i = symbols_[i].next) {
  1564. auto & symbol = symbols_[i];
  1565. auto token = vocab_.token_to_id.find(std::string(symbol.text, symbol.n));
  1566. if (token == vocab_.token_to_id.end()) {
  1567. // output any symbols that did not form tokens as bytes.
  1568. for (int j = 0; j < (int) symbol.n; ++j) {
  1569. llama_vocab::id token_id = static_cast<uint8_t>(symbol.text[j]) + 3;
  1570. output.push_back(token_id);
  1571. }
  1572. } else {
  1573. output.push_back((*token).second);
  1574. }
  1575. }
  1576. }
  1577. private:
  1578. void try_add_bigram(int left, int right) {
  1579. if (left == -1 || right == -1) {
  1580. return;
  1581. }
  1582. const std::string text = std::string(symbols_[left].text, symbols_[left].n + symbols_[right].n);
  1583. auto token = vocab_.token_to_id.find(text);
  1584. if (token == vocab_.token_to_id.end()) {
  1585. return;
  1586. }
  1587. if (static_cast<size_t>((*token).second) >= vocab_.id_to_token.size()) {
  1588. return;
  1589. }
  1590. const auto &tok_score = vocab_.id_to_token[(*token).second];
  1591. llama_sp_bigram bigram;
  1592. bigram.left = left;
  1593. bigram.right = right;
  1594. bigram.score = tok_score.score;
  1595. bigram.size = text.size();
  1596. work_queue_.push(bigram);
  1597. }
  1598. const llama_vocab & vocab_;
  1599. std::vector<llama_sp_symbol> symbols_;
  1600. llama_sp_bigram::queue work_queue_;
  1601. };
  1602. static std::vector<llama_vocab::id> llama_tokenize(const llama_vocab & vocab, const std::string & text, bool bos) {
  1603. llama_tokenizer tokenizer(vocab);
  1604. std::vector<llama_vocab::id> output;
  1605. if (text.empty()) {
  1606. return output;
  1607. }
  1608. if (bos) {
  1609. output.push_back(llama_token_bos());
  1610. }
  1611. tokenizer.tokenize(text, output);
  1612. return output;
  1613. }
  1614. //
  1615. // sampling
  1616. //
  1617. void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates) {
  1618. assert(candidates->size > 0);
  1619. const int64_t t_start_sample_us = ggml_time_us();
  1620. // Sort the logits in descending order
  1621. if (!candidates->sorted) {
  1622. std::sort(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) {
  1623. return a.logit > b.logit;
  1624. });
  1625. candidates->sorted = true;
  1626. }
  1627. float max_l = candidates->data[0].logit;
  1628. float cum_sum = 0.0f;
  1629. for (size_t i = 0; i < candidates->size; ++i) {
  1630. float p = expf(candidates->data[i].logit - max_l);
  1631. candidates->data[i].p = p;
  1632. cum_sum += p;
  1633. }
  1634. for (size_t i = 0; i < candidates->size; ++i) {
  1635. candidates->data[i].p /= cum_sum;
  1636. }
  1637. if (ctx) {
  1638. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1639. }
  1640. }
  1641. void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int k, size_t min_keep) {
  1642. const int64_t t_start_sample_us = ggml_time_us();
  1643. k = std::max(k, (int) min_keep);
  1644. k = std::min(k, (int) candidates->size);
  1645. // Sort scores in descending order
  1646. if (!candidates->sorted) {
  1647. auto comp = [](const llama_token_data & a, const llama_token_data & b) {
  1648. return a.logit > b.logit;
  1649. };
  1650. if (k == (int) candidates->size) {
  1651. std::sort(candidates->data, candidates->data + candidates->size, comp);
  1652. } else {
  1653. std::partial_sort(candidates->data, candidates->data + k, candidates->data + candidates->size, comp);
  1654. }
  1655. candidates->sorted = true;
  1656. }
  1657. candidates->size = k;
  1658. if (ctx) {
  1659. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1660. }
  1661. }
  1662. void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) {
  1663. if (p >= 1.0f) {
  1664. return;
  1665. }
  1666. llama_sample_softmax(ctx, candidates);
  1667. const int64_t t_start_sample_us = ggml_time_us();
  1668. // Compute the cumulative probabilities
  1669. float cum_sum = 0.0f;
  1670. size_t last_idx = candidates->size;
  1671. for (size_t i = 0; i < candidates->size; ++i) {
  1672. cum_sum += candidates->data[i].p;
  1673. // Check if the running sum is at least p or if we have kept at least min_keep tokens
  1674. // we set the last index to i+1 to indicate that the current iterate should be included in the set
  1675. if (cum_sum >= p && i + 1 >= min_keep) {
  1676. last_idx = i + 1;
  1677. break;
  1678. }
  1679. }
  1680. // Resize the output vector to keep only the top-p tokens
  1681. candidates->size = last_idx;
  1682. if (ctx) {
  1683. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1684. }
  1685. }
  1686. void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep) {
  1687. if (z >= 1.0f || candidates->size <= 2) {
  1688. return;
  1689. }
  1690. llama_sample_softmax(nullptr, candidates);
  1691. const int64_t t_start_sample_us = ggml_time_us();
  1692. // Compute the first and second derivatives
  1693. std::vector<float> first_derivatives(candidates->size - 1);
  1694. std::vector<float> second_derivatives(candidates->size - 2);
  1695. for (size_t i = 0; i < first_derivatives.size(); ++i) {
  1696. first_derivatives[i] = candidates->data[i].p - candidates->data[i + 1].p;
  1697. }
  1698. for (size_t i = 0; i < second_derivatives.size(); ++i) {
  1699. second_derivatives[i] = first_derivatives[i] - first_derivatives[i + 1];
  1700. }
  1701. // Calculate absolute value of second derivatives
  1702. for (size_t i = 0; i < second_derivatives.size(); ++i) {
  1703. second_derivatives[i] = abs(second_derivatives[i]);
  1704. }
  1705. // Normalize the second derivatives
  1706. float second_derivatives_sum = std::accumulate(second_derivatives.begin(), second_derivatives.end(), 0.0f);
  1707. for (float & value : second_derivatives) {
  1708. value /= second_derivatives_sum;
  1709. }
  1710. float cum_sum = 0.0f;
  1711. size_t last_idx = candidates->size;
  1712. for (size_t i = 0; i < second_derivatives.size(); ++i) {
  1713. cum_sum += second_derivatives[i];
  1714. // Check if the running sum is greater than z or if we have kept at least min_keep tokens
  1715. if (cum_sum > z && i >= min_keep) {
  1716. last_idx = i;
  1717. break;
  1718. }
  1719. }
  1720. // Resize the output vector to keep only the tokens above the tail location
  1721. candidates->size = last_idx;
  1722. if (ctx) {
  1723. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1724. }
  1725. }
  1726. void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) {
  1727. // Reference implementation:
  1728. // https://github.com/huggingface/transformers/compare/main...cimeister:typical-sampling:typical-pr
  1729. if (p >= 1.0f) {
  1730. return;
  1731. }
  1732. // Compute the softmax of logits and calculate entropy
  1733. llama_sample_softmax(nullptr, candidates);
  1734. const int64_t t_start_sample_us = ggml_time_us();
  1735. float entropy = 0.0f;
  1736. for (size_t i = 0; i < candidates->size; ++i) {
  1737. entropy += -candidates->data[i].p * logf(candidates->data[i].p);
  1738. }
  1739. // Compute the absolute difference between negative log probability and entropy for each candidate
  1740. std::vector<float> shifted_scores;
  1741. for (size_t i = 0; i < candidates->size; ++i) {
  1742. float shifted_score = fabsf(-logf(candidates->data[i].p) - entropy);
  1743. shifted_scores.push_back(shifted_score);
  1744. }
  1745. // Sort tokens based on the shifted_scores and their corresponding indices
  1746. std::vector<size_t> indices(candidates->size);
  1747. std::iota(indices.begin(), indices.end(), 0);
  1748. std::sort(indices.begin(), indices.end(), [&](size_t a, size_t b) {
  1749. return shifted_scores[a] < shifted_scores[b];
  1750. });
  1751. // Compute the cumulative probabilities
  1752. float cum_sum = 0.0f;
  1753. size_t last_idx = indices.size();
  1754. for (size_t i = 0; i < indices.size(); ++i) {
  1755. size_t idx = indices[i];
  1756. cum_sum += candidates->data[idx].p;
  1757. // Check if the running sum is greater than typical or if we have kept at least min_keep tokens
  1758. if (cum_sum > p && i >= min_keep - 1) {
  1759. last_idx = i + 1;
  1760. break;
  1761. }
  1762. }
  1763. // Resize the output vector to keep only the locally typical tokens
  1764. std::vector<llama_token_data> new_candidates;
  1765. for (size_t i = 0; i < last_idx; ++i) {
  1766. size_t idx = indices[i];
  1767. new_candidates.push_back(candidates->data[idx]);
  1768. }
  1769. // Replace the data in candidates with the new_candidates data
  1770. std::copy(new_candidates.begin(), new_candidates.end(), candidates->data);
  1771. candidates->size = new_candidates.size();
  1772. if (ctx) {
  1773. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1774. }
  1775. }
  1776. void llama_sample_temperature(struct llama_context * ctx, llama_token_data_array * candidates_p, float temp) {
  1777. const int64_t t_start_sample_us = ggml_time_us();
  1778. for (size_t i = 0; i < candidates_p->size; ++i) {
  1779. candidates_p->data[i].logit /= temp;
  1780. }
  1781. if (ctx) {
  1782. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1783. }
  1784. }
  1785. void llama_sample_repetition_penalty(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float penalty) {
  1786. if (last_tokens_size == 0 || penalty == 1.0f) {
  1787. return;
  1788. }
  1789. const int64_t t_start_sample_us = ggml_time_us();
  1790. for (size_t i = 0; i < candidates->size; ++i) {
  1791. const auto * token_iter = std::find(last_tokens, last_tokens + last_tokens_size, candidates->data[i].id);
  1792. if (token_iter == last_tokens + last_tokens_size) {
  1793. continue;
  1794. }
  1795. // The academic publication that described this technique actually just only divided, but that would cause tokens with negative logits to become more likely, which is obviously wrong.
  1796. // This is common fix for this problem, which is to multiply by the penalty instead of dividing.
  1797. if (candidates->data[i].logit <= 0) {
  1798. candidates->data[i].logit *= penalty;
  1799. } else {
  1800. candidates->data[i].logit /= penalty;
  1801. }
  1802. }
  1803. candidates->sorted = false;
  1804. if (ctx) {
  1805. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1806. }
  1807. }
  1808. void llama_sample_frequency_and_presence_penalties(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens_p, size_t last_tokens_size, float alpha_frequency, float alpha_presence) {
  1809. if (last_tokens_size == 0 || (alpha_frequency == 0.0f && alpha_presence == 0.0f)) {
  1810. return;
  1811. }
  1812. const int64_t t_start_sample_us = ggml_time_us();
  1813. // Create a frequency map to count occurrences of each token in last_tokens
  1814. std::unordered_map<llama_token, int> token_count;
  1815. for (size_t i = 0; i < last_tokens_size; ++i) {
  1816. token_count[last_tokens_p[i]]++;
  1817. }
  1818. // Apply frequency and presence penalties to the candidates
  1819. for (size_t i = 0; i < candidates->size; ++i) {
  1820. auto token_iter = token_count.find(candidates->data[i].id);
  1821. if (token_iter == token_count.end()) {
  1822. continue;
  1823. }
  1824. int count = token_iter->second;
  1825. candidates->data[i].logit -= float(count) * alpha_frequency + float(count > 0) * alpha_presence;
  1826. }
  1827. candidates->sorted = false;
  1828. if (ctx) {
  1829. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1830. }
  1831. }
  1832. static void llama_log_softmax(float * array, size_t size) {
  1833. float max_l = *std::max_element(array, array + size);
  1834. float sum = 0.f;
  1835. for (size_t i = 0; i < size; ++i) {
  1836. float p = expf(array[i] - max_l);
  1837. sum += p;
  1838. array[i] = p;
  1839. }
  1840. for (size_t i = 0; i < size; ++i) {
  1841. array[i] = logf(array[i] / sum);
  1842. }
  1843. }
  1844. void llama_sample_classifier_free_guidance(
  1845. struct llama_context * ctx,
  1846. llama_token_data_array * candidates,
  1847. struct llama_context * guidance_ctx,
  1848. float scale,
  1849. float smooth_factor) {
  1850. int64_t t_start_sample_us = t_start_sample_us = ggml_time_us();
  1851. assert(ctx);
  1852. auto n_vocab = llama_n_vocab(ctx);
  1853. assert(n_vocab == (int)candidates->size);
  1854. assert(!candidates->sorted);
  1855. std::vector<float> logits_base;
  1856. logits_base.reserve(candidates->size);
  1857. for (size_t i = 0; i < candidates->size; ++i) {
  1858. logits_base.push_back(candidates->data[i].logit);
  1859. }
  1860. llama_log_softmax(logits_base.data(), candidates->size);
  1861. float* logits_guidance = llama_get_logits(guidance_ctx);
  1862. llama_log_softmax(logits_guidance, n_vocab);
  1863. for (int i = 0; i < n_vocab; ++i) {
  1864. float logit_guidance = logits_guidance[i];
  1865. float logit_base = logits_base[i];
  1866. logits_guidance[i] = scale * (logit_base - logit_guidance) + logit_guidance;
  1867. }
  1868. llama_log_softmax(logits_guidance, n_vocab);
  1869. for (int i = 0; i < n_vocab; ++i) {
  1870. float logit_base = logits_base[i];
  1871. float logit_guidance = logits_guidance[i];
  1872. candidates->data[i].logit = smooth_factor * logit_guidance + (1.f - smooth_factor) * logit_base;
  1873. }
  1874. if (ctx) {
  1875. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1876. }
  1877. }
  1878. llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int m, float * mu) {
  1879. assert(ctx);
  1880. auto N = float(llama_n_vocab(ctx));
  1881. int64_t t_start_sample_us;
  1882. t_start_sample_us = ggml_time_us();
  1883. llama_sample_softmax(nullptr, candidates);
  1884. // Estimate s_hat using the most probable m tokens
  1885. float s_hat = 0.0;
  1886. float sum_ti_bi = 0.0;
  1887. float sum_ti_sq = 0.0;
  1888. for (size_t i = 0; i < size_t(m - 1) && i < candidates->size - 1; ++i) {
  1889. float t_i = logf(float(i + 2) / float(i + 1));
  1890. float b_i = logf(candidates->data[i].p / candidates->data[i + 1].p);
  1891. sum_ti_bi += t_i * b_i;
  1892. sum_ti_sq += t_i * t_i;
  1893. }
  1894. s_hat = sum_ti_bi / sum_ti_sq;
  1895. // Compute k from the estimated s_hat and target surprise value
  1896. float epsilon_hat = s_hat - 1;
  1897. float k = powf((epsilon_hat * powf(2, *mu)) / (1 - powf(N, -epsilon_hat)), 1 / s_hat);
  1898. // Sample the next word X using top-k sampling
  1899. llama_sample_top_k(nullptr, candidates, int(k), 1);
  1900. if (ctx) {
  1901. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1902. }
  1903. llama_token X = llama_sample_token(ctx, candidates);
  1904. t_start_sample_us = ggml_time_us();
  1905. // Compute error as the difference between observed surprise and target surprise value
  1906. size_t X_idx = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) {
  1907. return candidate.id == X;
  1908. }));
  1909. float observed_surprise = -log2f(candidates->data[X_idx].p);
  1910. float e = observed_surprise - tau;
  1911. // Update mu using the learning rate and error
  1912. *mu = *mu - eta * e;
  1913. if (ctx) {
  1914. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1915. }
  1916. return X;
  1917. }
  1918. llama_token llama_sample_token_mirostat_v2(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, float * mu) {
  1919. int64_t t_start_sample_us;
  1920. t_start_sample_us = ggml_time_us();
  1921. llama_sample_softmax(ctx, candidates);
  1922. // Truncate the words with surprise values greater than mu
  1923. candidates->size = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) {
  1924. return -log2f(candidate.p) > *mu;
  1925. }));
  1926. if (candidates->size == 0) {
  1927. candidates->size = 1;
  1928. }
  1929. if (ctx) {
  1930. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1931. }
  1932. // Normalize the probabilities of the remaining words
  1933. llama_sample_softmax(ctx, candidates);
  1934. // Sample the next word X from the remaining words
  1935. llama_token X = llama_sample_token(ctx, candidates);
  1936. t_start_sample_us = ggml_time_us();
  1937. // Compute error as the difference between observed surprise and target surprise value
  1938. size_t X_idx = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) {
  1939. return candidate.id == X;
  1940. }));
  1941. float observed_surprise = -log2f(candidates->data[X_idx].p);
  1942. float e = observed_surprise - tau;
  1943. // Update mu using the learning rate and error
  1944. *mu = *mu - eta * e;
  1945. if (ctx) {
  1946. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1947. }
  1948. return X;
  1949. }
  1950. llama_token llama_sample_token_greedy(struct llama_context * ctx, llama_token_data_array * candidates) {
  1951. const int64_t t_start_sample_us = ggml_time_us();
  1952. // Find max element
  1953. auto * max_iter = std::max_element(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) {
  1954. return a.logit < b.logit;
  1955. });
  1956. llama_token result = max_iter->id;
  1957. if (ctx) {
  1958. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1959. ctx->n_sample++;
  1960. }
  1961. return result;
  1962. }
  1963. llama_token llama_sample_token(struct llama_context * ctx, llama_token_data_array * candidates) {
  1964. assert(ctx);
  1965. const int64_t t_start_sample_us = ggml_time_us();
  1966. llama_sample_softmax(nullptr, candidates);
  1967. std::vector<float> probs;
  1968. probs.reserve(candidates->size);
  1969. for (size_t i = 0; i < candidates->size; ++i) {
  1970. probs.push_back(candidates->data[i].p);
  1971. }
  1972. std::discrete_distribution<> dist(probs.begin(), probs.end());
  1973. auto & rng = ctx->rng;
  1974. int idx = dist(rng);
  1975. llama_token result = candidates->data[idx].id;
  1976. ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
  1977. ctx->n_sample++;
  1978. return result;
  1979. }
  1980. //
  1981. // quantization
  1982. //
  1983. static void llama_convert_tensor_internal(const llama_load_tensor & tensor, llama_buffer & output, const int nelements, const int nthread) {
  1984. if (output.size < nelements * sizeof(float)) {
  1985. output.resize(nelements * sizeof(float));
  1986. }
  1987. float * f32_output = (float *) output.addr;
  1988. ggml_type_traits_t qtype;
  1989. if (ggml_is_quantized(tensor.type)) {
  1990. qtype = ggml_internal_get_type_traits(tensor.type);
  1991. if (qtype.to_float == NULL) {
  1992. throw std::runtime_error(format("type %s unsupported for integer quantization: no dequantization available", ggml_type_name(tensor.type)));
  1993. }
  1994. } else if (tensor.type != GGML_TYPE_F16) {
  1995. throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(tensor.type)));
  1996. }
  1997. if (nthread < 2) {
  1998. if (tensor.type == GGML_TYPE_F16) {
  1999. ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor.data, f32_output, nelements);
  2000. } else if (ggml_is_quantized(tensor.type)) {
  2001. qtype.to_float(tensor.data, f32_output, nelements);
  2002. } else {
  2003. LLAMA_ASSERT(false); // unreachable
  2004. }
  2005. return;
  2006. }
  2007. auto block_size = tensor.type == GGML_TYPE_F16 ? 1 : (size_t)ggml_blck_size(tensor.type);
  2008. auto block_size_bytes = ggml_type_size(tensor.type);
  2009. LLAMA_ASSERT(nelements % block_size == 0);
  2010. auto nblocks = nelements / block_size;
  2011. auto blocks_per_thread = nblocks / nthread;
  2012. auto spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count
  2013. std::vector<std::thread> workers;
  2014. for (auto tnum = 0, in_buff_offs = 0, out_buff_offs = 0; tnum < nthread; tnum++) {
  2015. auto thr_blocks = blocks_per_thread + (tnum == nthread - 1 ? spare_blocks : 0); // num blocks for this thread
  2016. auto thr_elems = thr_blocks * block_size; // number of elements for this thread
  2017. auto thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread
  2018. auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) {
  2019. if (typ == GGML_TYPE_F16) {
  2020. ggml_fp16_to_fp32_row((ggml_fp16_t *)inbuf, outbuf, nels);
  2021. } else {
  2022. qtype.to_float(inbuf, outbuf, nels);
  2023. }
  2024. };
  2025. workers.push_back(std::thread(compute, tensor.type, tensor.data + in_buff_offs, f32_output + out_buff_offs, thr_elems));
  2026. in_buff_offs += thr_block_bytes;
  2027. out_buff_offs += thr_elems;
  2028. }
  2029. for (auto & worker : workers) {
  2030. worker.join();
  2031. }
  2032. }
  2033. static void llama_model_quantize_internal(const std::string & fname_inp, const std::string & fname_out, const llama_model_quantize_params * params) {
  2034. ggml_type quantized_type;
  2035. llama_ftype ftype = params->ftype;
  2036. int nthread = params->nthread;
  2037. switch (params->ftype) {
  2038. case LLAMA_FTYPE_MOSTLY_Q4_0: quantized_type = GGML_TYPE_Q4_0; break;
  2039. case LLAMA_FTYPE_MOSTLY_Q4_1: quantized_type = GGML_TYPE_Q4_1; break;
  2040. case LLAMA_FTYPE_MOSTLY_Q5_0: quantized_type = GGML_TYPE_Q5_0; break;
  2041. case LLAMA_FTYPE_MOSTLY_Q5_1: quantized_type = GGML_TYPE_Q5_1; break;
  2042. case LLAMA_FTYPE_MOSTLY_Q8_0: quantized_type = GGML_TYPE_Q8_0; break;
  2043. case LLAMA_FTYPE_MOSTLY_F16: quantized_type = GGML_TYPE_F16; break;
  2044. case LLAMA_FTYPE_ALL_F32: quantized_type = GGML_TYPE_F32; break;
  2045. #ifdef GGML_USE_K_QUANTS
  2046. // K-quants
  2047. case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break;
  2048. case LLAMA_FTYPE_MOSTLY_Q3_K_S:
  2049. case LLAMA_FTYPE_MOSTLY_Q3_K_M:
  2050. case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break;
  2051. case LLAMA_FTYPE_MOSTLY_Q4_K_S:
  2052. case LLAMA_FTYPE_MOSTLY_Q4_K_M: quantized_type = GGML_TYPE_Q4_K; break;
  2053. case LLAMA_FTYPE_MOSTLY_Q5_K_S:
  2054. case LLAMA_FTYPE_MOSTLY_Q5_K_M: quantized_type = GGML_TYPE_Q5_K; break;
  2055. case LLAMA_FTYPE_MOSTLY_Q6_K: quantized_type = GGML_TYPE_Q6_K; break;
  2056. #endif
  2057. default: throw std::runtime_error(format("invalid output file type %d\n", ftype));
  2058. }
  2059. if (nthread <= 0) {
  2060. nthread = std::thread::hardware_concurrency();
  2061. }
  2062. std::unique_ptr<llama_model_loader> model_loader(new llama_model_loader(fname_inp, /*use_mmap*/ false));
  2063. llama_file_saver file_saver(fname_out.c_str(), model_loader->file_loader.get(), params->ftype);
  2064. #ifdef GGML_USE_K_QUANTS
  2065. int n_attention_wv = 0;
  2066. int n_feed_forward_w2 = 0;
  2067. for (auto& tensor : model_loader->tensors_map.tensors) {
  2068. if (tensor.name.find("attention.wv.weight") != std::string::npos) {
  2069. ++n_attention_wv;
  2070. }
  2071. else if (tensor.name.find("feed_forward.w2.weight") != std::string::npos) {
  2072. ++n_feed_forward_w2;
  2073. }
  2074. }
  2075. int i_attention_wv = 0;
  2076. int i_feed_forward_w2 = 0;
  2077. #endif
  2078. size_t total_size_org = 0;
  2079. size_t total_size_new = 0;
  2080. std::vector<int64_t> hist_all(1 << 4, 0);
  2081. std::vector<std::thread> workers;
  2082. std::mutex mutex;
  2083. auto use_more_bits = [] (int i_layer, int num_layers) -> bool {
  2084. return i_layer < num_layers/8 || i_layer >= 7*num_layers/8 || (i_layer - num_layers/8)%3 == 2;
  2085. };
  2086. size_t idx = 0;
  2087. for (llama_load_tensor & tensor : model_loader->tensors_map.tensors) {
  2088. llama_buffer read_data;
  2089. read_data.resize(tensor.size);
  2090. tensor.data = read_data.addr;
  2091. model_loader->load_data_for(tensor);
  2092. printf("[%4zu/%4zu] %36s - %16s, type = %6s, ",
  2093. ++idx, model_loader->tensors_map.tensors.size(),
  2094. tensor.name.c_str(), llama_format_tensor_shape(tensor.ne).c_str(),
  2095. ggml_type_name(tensor.type));
  2096. // This used to be a regex, but <regex> has an extreme cost to compile times.
  2097. bool quantize = tensor.name.rfind("weight") == tensor.name.size() - 6; // ends with 'weight'?
  2098. // quantize only 2D tensors
  2099. quantize &= (tensor.ne.size() == 2);
  2100. quantize &= params->quantize_output_tensor || tensor.name != "output.weight";
  2101. quantize &= quantized_type != tensor.type;
  2102. enum ggml_type new_type;
  2103. void * new_data;
  2104. size_t new_size;
  2105. llama_buffer work;
  2106. if (!quantize) {
  2107. new_type = tensor.type;
  2108. new_data = tensor.data;
  2109. new_size = tensor.size;
  2110. printf("size = %8.3f MB\n", tensor.size/1024.0/1024.0);
  2111. } else {
  2112. new_type = quantized_type;
  2113. #ifdef GGML_USE_K_QUANTS
  2114. bool convert_incompatible_tensor = false;
  2115. if (quantized_type == GGML_TYPE_Q2_K || quantized_type == GGML_TYPE_Q3_K || quantized_type == GGML_TYPE_Q4_K ||
  2116. quantized_type == GGML_TYPE_Q5_K || quantized_type == GGML_TYPE_Q6_K) {
  2117. int nx = tensor.ne.at(0);
  2118. int ny = tensor.ne.at(1);
  2119. if (nx % QK_K != 0 || ny % QK_K != 0) {
  2120. fprintf(stderr, "\n\nTensor sizes %d x %d are not divisible by %d, required for k-quants.\n",nx,ny,QK_K);
  2121. convert_incompatible_tensor = true;
  2122. }
  2123. }
  2124. if (tensor.name == "output.weight") {
  2125. int nx = tensor.ne.at(0);
  2126. int ny = tensor.ne.at(1);
  2127. if (nx % QK_K == 0 && ny % QK_K == 0) {
  2128. new_type = GGML_TYPE_Q6_K;
  2129. }
  2130. } else if (tensor.name.find("attention.wv.weight") != std::string::npos) {
  2131. if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K;
  2132. else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;
  2133. else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) &&
  2134. use_more_bits(i_attention_wv, n_attention_wv)) new_type = GGML_TYPE_Q6_K;
  2135. else if (QK_K == 64 && (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S) &&
  2136. (i_attention_wv < n_attention_wv/8 || i_attention_wv >= 7*n_attention_wv/8)) new_type = GGML_TYPE_Q6_K;
  2137. ++i_attention_wv;
  2138. } else if (tensor.name.find("feed_forward.w2.weight") != std::string::npos) {
  2139. if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K;
  2140. else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;
  2141. else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) &&
  2142. use_more_bits(i_feed_forward_w2, n_feed_forward_w2)) new_type = GGML_TYPE_Q6_K;
  2143. //else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && i_feed_forward_w2 < n_feed_forward_w2/8) new_type = GGML_TYPE_Q6_K;
  2144. ++i_feed_forward_w2;
  2145. } else if (tensor.name.find("attention.wo.weight") != std::string::npos) {
  2146. if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K;
  2147. else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;
  2148. }
  2149. if (convert_incompatible_tensor) {
  2150. if (tensor.name == "output.weight") {
  2151. new_type = GGML_TYPE_F16; //fall back to F16 instead of just failing.
  2152. fprintf(stderr, "F16 will be used for this tensor instead.\n");
  2153. } else if (tensor.name == "tok_embeddings.weight") {
  2154. new_type = GGML_TYPE_Q4_0; //fall back to Q4_0 instead of just failing.
  2155. fprintf(stderr, "Q4_0 will be used for this tensor instead.\n");
  2156. } else {
  2157. throw std::runtime_error("Unsupported tensor size encountered\n");
  2158. }
  2159. }
  2160. #endif
  2161. float * f32_data;
  2162. size_t nelements = tensor.ne.at(0) * tensor.ne.at(1);
  2163. llama_buffer f32_conv_buf;
  2164. if (tensor.type == GGML_TYPE_F32) {
  2165. f32_data = (float *) tensor.data;
  2166. } else if (ggml_is_quantized(tensor.type) && !params->allow_requantize) {
  2167. throw std::runtime_error(format("requantizing from type %s is disabled", ggml_type_name(tensor.type)));
  2168. } else {
  2169. llama_convert_tensor_internal(tensor, f32_conv_buf, nelements, nthread);
  2170. f32_data = (float *) f32_conv_buf.addr;
  2171. }
  2172. printf("quantizing .. ");
  2173. fflush(stdout);
  2174. work.resize(nelements * 4); // upper bound on size
  2175. new_data = work.addr;
  2176. std::vector<int64_t> hist_cur(1 << 4, 0);
  2177. int chunk_size = 32 * 512;
  2178. const int nchunk = (nelements + chunk_size - 1)/chunk_size;
  2179. const int nthread_use = nthread > 1 ? std::max(1, std::min(nthread, nchunk)) : 1;
  2180. if (nthread_use < 2) {
  2181. new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nelements, hist_cur.data());
  2182. } else {
  2183. size_t counter = 0;
  2184. new_size = 0;
  2185. auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, nelements, chunk_size] () {
  2186. std::vector<int64_t> local_hist;
  2187. size_t local_size = 0;
  2188. while (true) {
  2189. std::unique_lock<std::mutex> lock(mutex);
  2190. size_t first = counter; counter += chunk_size;
  2191. if (first >= nelements) {
  2192. if (!local_hist.empty()) {
  2193. for (int j=0; j<int(local_hist.size()); ++j) {
  2194. hist_cur[j] += local_hist[j];
  2195. }
  2196. new_size += local_size;
  2197. }
  2198. break;
  2199. }
  2200. lock.unlock();
  2201. size_t last = std::min(nelements, first + chunk_size);
  2202. if (local_hist.empty()) {
  2203. local_hist.resize(hist_cur.size(), 0);
  2204. }
  2205. local_size += ggml_quantize_chunk(new_type, f32_data, new_data, first, last - first, local_hist.data());
  2206. }
  2207. };
  2208. if ((int) workers.size() < nthread_use - 1) {
  2209. workers.resize(nthread_use - 1);
  2210. }
  2211. for (int it = 0; it < nthread_use - 1; ++it) {
  2212. workers[it] = std::thread(compute);
  2213. }
  2214. compute();
  2215. for (int it = 0; it < nthread_use - 1; ++it) {
  2216. workers[it].join();
  2217. }
  2218. }
  2219. printf("size = %8.2f MB -> %8.2f MB | hist: ", tensor.size/1024.0/1024.0, new_size/1024.0/1024.0);
  2220. int64_t tot_count = 0;
  2221. for (size_t i = 0; i < hist_cur.size(); i++) {
  2222. hist_all[i] += hist_cur[i];
  2223. tot_count += hist_cur[i];
  2224. }
  2225. if (tot_count > 0) {
  2226. for (size_t i = 0; i < hist_cur.size(); i++) {
  2227. printf("%5.3f ", hist_cur[i] / float(nelements));
  2228. }
  2229. }
  2230. printf("\n");
  2231. }
  2232. total_size_org += tensor.size;
  2233. total_size_new += new_size;
  2234. file_saver.write_tensor(tensor, new_type, new_data, new_size);
  2235. }
  2236. printf("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0);
  2237. printf("%s: quant size = %8.2f MB\n", __func__, total_size_new/1024.0/1024.0);
  2238. {
  2239. int64_t sum_all = 0;
  2240. for (size_t i = 0; i < hist_all.size(); i++) {
  2241. sum_all += hist_all[i];
  2242. }
  2243. if (sum_all > 0) {
  2244. printf("%s: hist: ", __func__);
  2245. for (size_t i = 0; i < hist_all.size(); i++) {
  2246. printf("%5.3f ", hist_all[i] / float(sum_all));
  2247. }
  2248. printf("\n");
  2249. }
  2250. }
  2251. }
  2252. //
  2253. // interface implementation
  2254. //
  2255. struct llama_model * llama_load_model_from_file(
  2256. const char * path_model,
  2257. struct llama_context_params params) {
  2258. ggml_time_init();
  2259. llama_model * model = new llama_model;
  2260. ggml_type memory_type = params.f16_kv ? GGML_TYPE_F16 : GGML_TYPE_F32;
  2261. if (!llama_model_load(path_model, *model, model->vocab, params.n_ctx, params.n_batch, params.n_gpu_layers,
  2262. params.main_gpu, params.tensor_split, params.low_vram, memory_type, params.use_mmap, params.use_mlock,
  2263. params.vocab_only, params.progress_callback, params.progress_callback_user_data)) {
  2264. delete model;
  2265. fprintf(stderr, "%s: failed to load model\n", __func__);
  2266. return nullptr;
  2267. }
  2268. return model;
  2269. }
  2270. void llama_free_model(struct llama_model * model) {
  2271. delete model;
  2272. }
  2273. struct llama_context * llama_new_context_with_model(
  2274. struct llama_model * model,
  2275. struct llama_context_params params) {
  2276. if (!model) {
  2277. return nullptr;
  2278. }
  2279. llama_context * ctx = new llama_context(*model, model->vocab);
  2280. if (params.seed == LLAMA_DEFAULT_SEED) {
  2281. params.seed = time(NULL);
  2282. }
  2283. unsigned cur_percentage = 0;
  2284. if (params.progress_callback == NULL) {
  2285. params.progress_callback_user_data = &cur_percentage;
  2286. params.progress_callback = [](float progress, void * ctx) {
  2287. unsigned * cur_percentage_p = (unsigned *) ctx;
  2288. unsigned percentage = (unsigned) (100 * progress);
  2289. while (percentage > *cur_percentage_p) {
  2290. *cur_percentage_p = percentage;
  2291. fprintf(stderr, ".");
  2292. fflush(stderr);
  2293. if (percentage >= 100) {
  2294. fprintf(stderr, "\n");
  2295. }
  2296. }
  2297. };
  2298. }
  2299. ctx->rng = std::mt19937(params.seed);
  2300. ctx->logits_all = params.logits_all;
  2301. ggml_type memory_type = params.f16_kv ? GGML_TYPE_F16 : GGML_TYPE_F32;
  2302. // reserve memory for context buffers
  2303. if (!params.vocab_only) {
  2304. if (!kv_cache_init(ctx->model.hparams, ctx->kv_self, memory_type, ctx->model.hparams.n_ctx, params.n_gpu_layers)) {
  2305. fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__);
  2306. llama_free(ctx);
  2307. return nullptr;
  2308. }
  2309. {
  2310. const size_t memory_size = ggml_nbytes(ctx->kv_self.k) + ggml_nbytes(ctx->kv_self.v);
  2311. fprintf(stderr, "%s: kv self size = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0);
  2312. }
  2313. const auto & hparams = ctx->model.hparams;
  2314. // resized during inference
  2315. if (params.logits_all) {
  2316. ctx->logits.reserve(hparams.n_ctx*hparams.n_vocab);
  2317. } else {
  2318. ctx->logits.reserve(hparams.n_vocab);
  2319. }
  2320. if (params.embedding){
  2321. ctx->embedding.resize(hparams.n_embd);
  2322. }
  2323. ctx->buf_compute.resize(MEM_REQ_EVAL().at(ctx->model.type));
  2324. ctx->buf_scratch[0].resize(MEM_REQ_SCRATCH0().at(ctx->model.type));
  2325. ctx->buf_scratch[1].resize(MEM_REQ_SCRATCH1().at(ctx->model.type));
  2326. }
  2327. #ifdef GGML_USE_METAL
  2328. if (params.n_gpu_layers > 0) {
  2329. // this allocates all Metal resources and memory buffers
  2330. ctx->ctx_metal = ggml_metal_init(1);
  2331. void * data_ptr = NULL;
  2332. size_t data_size = 0;
  2333. if (params.use_mmap) {
  2334. data_ptr = ctx->model.mapping->addr;
  2335. data_size = ctx->model.mapping->size;
  2336. } else {
  2337. data_ptr = ggml_get_mem_buffer(ctx->model.ctx);
  2338. data_size = ggml_get_mem_size (ctx->model.ctx);
  2339. }
  2340. const size_t max_size = ggml_get_max_tensor_size(ctx->model.ctx);
  2341. printf("%s: max tensor size = %8.2f MB\n", __func__, max_size/1024.0/1024.0);
  2342. #define LLAMA_METAL_CHECK_BUF(result) \
  2343. if (!(result)) { \
  2344. fprintf(stderr, "%s: failed to add buffer\n", __func__); \
  2345. llama_free(ctx); \
  2346. return NULL; \
  2347. }
  2348. LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "data", data_ptr, data_size, max_size));
  2349. LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "eval", ctx->buf_compute.addr, ctx->buf_compute.size, 0));
  2350. LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "kv", ctx->kv_self.buf.addr, ctx->kv_self.buf.size, 0));
  2351. LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "scr0", ctx->buf_scratch[0].addr, ctx->buf_scratch[0].size, 0));
  2352. LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "scr1", ctx->buf_scratch[1].addr, ctx->buf_scratch[1].size, 0));
  2353. #undef LLAMA_METAL_CHECK_BUF
  2354. }
  2355. #endif
  2356. #ifdef GGML_USE_MPI
  2357. ctx->ctx_mpi = ggml_mpi_init();
  2358. if (ggml_mpi_rank(ctx->ctx_mpi) > 0) {
  2359. // Enter a blocking eval loop with dummy input, letting rank=0 drive the process
  2360. const std::vector<llama_token> tmp(ctx->model.hparams.n_ctx, llama_token_bos());
  2361. while (!llama_eval(ctx, tmp.data(), tmp.size(), 0, 0)) {};
  2362. llama_backend_free();
  2363. exit(1);
  2364. }
  2365. #endif
  2366. return ctx;
  2367. }
  2368. struct llama_context * llama_init_from_file(
  2369. const char * path_model,
  2370. struct llama_context_params params) {
  2371. struct llama_model * model = llama_load_model_from_file(path_model, params);
  2372. if (!model) {
  2373. return nullptr;
  2374. }
  2375. struct llama_context * ctx = llama_new_context_with_model(model, params);
  2376. ctx->model_owner = true;
  2377. return ctx;
  2378. }
  2379. void llama_free(struct llama_context * ctx) {
  2380. if (ctx->model_owner) {
  2381. delete &ctx->model;
  2382. }
  2383. delete ctx;
  2384. }
  2385. int llama_model_quantize(
  2386. const char * fname_inp,
  2387. const char * fname_out,
  2388. const llama_model_quantize_params *params) {
  2389. try {
  2390. llama_model_quantize_internal(fname_inp, fname_out, params);
  2391. return 0;
  2392. } catch (const std::exception & err) {
  2393. fprintf(stderr, "%s: failed to quantize: %s\n", __func__, err.what());
  2394. return 1;
  2395. }
  2396. }
  2397. int llama_apply_lora_from_file_internal(const struct llama_model & model, const char * path_lora, const char * path_base_model, int n_threads) {
  2398. fprintf(stderr, "%s: applying lora adapter from '%s' - please wait ...\n", __func__, path_lora);
  2399. const int64_t t_start_lora_us = ggml_time_us();
  2400. auto fin = std::ifstream(path_lora, std::ios::binary);
  2401. if (!fin) {
  2402. fprintf(stderr, "%s: failed to open '%s'\n", __func__, path_lora);
  2403. return 1;
  2404. }
  2405. // verify magic and version
  2406. {
  2407. uint32_t magic;
  2408. fin.read((char *) &magic, sizeof(magic));
  2409. if (magic != LLAMA_FILE_MAGIC_GGLA) {
  2410. fprintf(stderr, "%s: bad file magic\n", __func__);
  2411. return 1;
  2412. }
  2413. uint32_t format_version;
  2414. fin.read((char *) &format_version, sizeof(format_version));
  2415. if (format_version != 1) {
  2416. fprintf(stderr, "%s: unsupported file version\n", __func__ );
  2417. return 1;
  2418. }
  2419. }
  2420. int32_t lora_r;
  2421. int32_t lora_alpha;
  2422. fin.read((char *) &lora_r, sizeof(lora_r));
  2423. fin.read((char *) &lora_alpha, sizeof(lora_alpha));
  2424. float scaling = (float)lora_alpha / (float)lora_r;
  2425. fprintf(stderr, "%s: r = %d, alpha = %d, scaling = %.2f\n", __func__, lora_r, lora_alpha, scaling);
  2426. // create a temporary ggml context to store the lora tensors
  2427. // todo: calculate size from biggest possible tensor
  2428. std::vector<uint8_t> lora_buf(1024ull * 1024ull * 1024ull);
  2429. struct ggml_init_params params;
  2430. params.mem_size = lora_buf.size();
  2431. params.mem_buffer = lora_buf.data();
  2432. params.no_alloc = false;
  2433. ggml_context * lora_ctx = ggml_init(params);
  2434. std::unordered_map<std::string, struct ggml_tensor *> lora_tensors;
  2435. // create a name -> tensor map of the model to accelerate lookups
  2436. std::unordered_map<std::string, struct ggml_tensor*> model_tensors;
  2437. for (const auto & kv: model.tensors_by_name) {
  2438. model_tensors.insert(kv);
  2439. }
  2440. // load base model
  2441. std::unique_ptr<llama_model_loader> model_loader;
  2442. ggml_context * base_ctx = NULL;
  2443. llama_buffer base_buf;
  2444. if (path_base_model) {
  2445. fprintf(stderr, "%s: loading base model from '%s'\n", __func__, path_base_model);
  2446. model_loader.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true));
  2447. size_t ctx_size;
  2448. size_t mmapped_size;
  2449. model_loader->calc_sizes(&ctx_size, &mmapped_size);
  2450. base_buf.resize(ctx_size);
  2451. ggml_init_params base_params;
  2452. base_params.mem_size = base_buf.size;
  2453. base_params.mem_buffer = base_buf.addr;
  2454. base_params.no_alloc = model_loader->use_mmap;
  2455. base_ctx = ggml_init(base_params);
  2456. model_loader->ggml_ctx = base_ctx;
  2457. // maybe this should in llama_model_loader
  2458. if (model_loader->use_mmap) {
  2459. model_loader->mapping.reset(new llama_mmap(&model_loader->file_loader->file, /* prefetch */ 0, ggml_is_numa()));
  2460. }
  2461. }
  2462. // read tensors and apply
  2463. bool warned = false;
  2464. int n_tensors = 0;
  2465. std::vector<uint8_t> work_buffer;
  2466. while (true) {
  2467. int32_t n_dims;
  2468. int32_t length;
  2469. int32_t ftype;
  2470. fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
  2471. fin.read(reinterpret_cast<char *>(&length), sizeof(length));
  2472. fin.read(reinterpret_cast<char *>(&ftype), sizeof(ftype));
  2473. if (fin.eof()) {
  2474. break;
  2475. }
  2476. int32_t ne[2] = { 1, 1 };
  2477. for (int i = 0; i < n_dims; ++i) {
  2478. fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
  2479. }
  2480. std::string name;
  2481. {
  2482. char buf[1024];
  2483. fin.read(buf, length);
  2484. name = std::string(buf, length);
  2485. }
  2486. // check for lora suffix and get the type of tensor
  2487. const std::string lora_suffix = ".lora";
  2488. size_t pos = name.rfind(lora_suffix);
  2489. if (pos == std::string::npos) {
  2490. fprintf(stderr, "%s: error: '%s' is not a lora tensor\n", __func__, name.c_str());
  2491. return 1;
  2492. }
  2493. std::string lora_type = name.substr(pos + lora_suffix.length());
  2494. std::string base_name = name;
  2495. base_name.erase(pos);
  2496. // fprintf(stderr, "%s: %s => %s (lora type %s) ", __func__, name.c_str(),base_name.c_str(), lora_type.c_str());
  2497. if (model_tensors.find(base_name) == model_tensors.end()) {
  2498. fprintf(stderr, "%s: unknown tensor '%s' in lora adapter\n", __func__, name.data());
  2499. return 1;
  2500. }
  2501. // create ggml tensor
  2502. ggml_type wtype;
  2503. switch (ftype) {
  2504. case 0: wtype = GGML_TYPE_F32; break;
  2505. case 1: wtype = GGML_TYPE_F16; break;
  2506. default:
  2507. {
  2508. fprintf(stderr, "%s: invalid tensor data type '%d'\n",
  2509. __func__, ftype);
  2510. return false;
  2511. }
  2512. }
  2513. ggml_tensor * lora_tensor;
  2514. if (n_dims == 2) {
  2515. lora_tensor = ggml_new_tensor_2d(lora_ctx, wtype, ne[0], ne[1]);
  2516. }
  2517. else {
  2518. fprintf(stderr, "%s: unsupported tensor dimension %d\n", __func__, n_dims);
  2519. return 1;
  2520. }
  2521. ggml_set_name(lora_tensor, "lora_tensor");
  2522. // load tensor data
  2523. size_t offset = fin.tellg();
  2524. size_t tensor_data_size = ggml_nbytes(lora_tensor);
  2525. offset = (offset + 31) & -32;
  2526. fin.seekg(offset);
  2527. fin.read((char*)lora_tensor->data, tensor_data_size);
  2528. lora_tensors[name] = lora_tensor;
  2529. // check if we have both A and B tensors and apply
  2530. if (lora_tensors.find(base_name + ".loraA") != lora_tensors.end() &&
  2531. lora_tensors.find(base_name + ".loraB") != lora_tensors.end()) {
  2532. ggml_tensor * dest_t = model_tensors[base_name];
  2533. offload_func_t offload_func = llama_nop;
  2534. offload_func_t offload_func_force_inplace = llama_nop;
  2535. #ifdef GGML_USE_CUBLAS
  2536. if (dest_t->backend == GGML_BACKEND_GPU || dest_t->backend == GGML_BACKEND_GPU_SPLIT) {
  2537. if (dest_t->type != GGML_TYPE_F16) {
  2538. throw std::runtime_error(format(
  2539. "%s: error: the simultaneous use of LoRAs and GPU acceleration is only supported for f16 models", __func__));
  2540. }
  2541. offload_func = ggml_cuda_assign_buffers;
  2542. offload_func_force_inplace = ggml_cuda_assign_buffers_force_inplace;
  2543. }
  2544. #endif // GGML_USE_CUBLAS
  2545. ggml_tensor * base_t;
  2546. if (model_loader) {
  2547. // load from base model
  2548. if (model_loader->tensors_map.name_to_idx.find(base_name) == model_loader->tensors_map.name_to_idx.end()) {
  2549. fprintf(stderr, "%s: error: tensor '%s' not found in base model\n", __func__, base_name.c_str());
  2550. return 1;
  2551. }
  2552. size_t idx = model_loader->tensors_map.name_to_idx[base_name];
  2553. llama_load_tensor & lt = model_loader->tensors_map.tensors[idx];
  2554. base_t = model_loader->get_tensor(base_name, { (uint32_t)dest_t->ne[0], (uint32_t)dest_t->ne[1] }, GGML_BACKEND_CPU);
  2555. lt.data = (uint8_t *) lt.ggml_tensor->data;
  2556. model_loader->load_data_for(lt);
  2557. lt.ggml_tensor->data = lt.data;
  2558. }
  2559. else {
  2560. base_t = dest_t;
  2561. }
  2562. if (ggml_is_quantized(base_t->type)) {
  2563. if (!warned) {
  2564. fprintf(stderr, "%s: warning: using a lora adapter with a quantized model may result in poor quality, "
  2565. "use a f16 or f32 base model with --lora-base\n", __func__);
  2566. warned = true;
  2567. }
  2568. }
  2569. ggml_tensor * loraA = lora_tensors[base_name + ".loraA"];
  2570. GGML_ASSERT(loraA->type == GGML_TYPE_F32);
  2571. ggml_set_name(loraA, "loraA");
  2572. ggml_tensor * loraB = lora_tensors[base_name + ".loraB"];
  2573. GGML_ASSERT(loraB->type == GGML_TYPE_F32);
  2574. ggml_set_name(loraB, "loraB");
  2575. if (base_t->ne[0] != loraA->ne[1] || base_t->ne[1] != loraB->ne[1]) {
  2576. fprintf(stderr, "%s: incompatible tensor dimensions (%" PRId64 " and %" PRId64 ");"
  2577. " are you sure that this adapter is for this model?\n", __func__, base_t->ne[0], loraA->ne[1]);
  2578. return 1;
  2579. }
  2580. // w = w + BA*s
  2581. ggml_tensor * BA = ggml_mul_mat(lora_ctx, loraA, loraB);
  2582. offload_func(BA);
  2583. ggml_set_name(BA, "BA");
  2584. if (scaling != 1.0f) {
  2585. ggml_tensor * scale_tensor = ggml_new_f32(lora_ctx, scaling);
  2586. ggml_set_name(scale_tensor, "scale_tensor");
  2587. BA = ggml_scale_inplace(lora_ctx, BA, scale_tensor);
  2588. offload_func(BA);
  2589. ggml_set_name(BA, "BA_scaled");
  2590. }
  2591. ggml_tensor * r;
  2592. if (base_t == dest_t) {
  2593. r = ggml_add_inplace(lora_ctx, dest_t, BA);
  2594. offload_func_force_inplace(r);
  2595. ggml_set_name(r, "r_add_inplace");
  2596. }
  2597. else {
  2598. r = ggml_add(lora_ctx, base_t, BA);
  2599. offload_func(r);
  2600. ggml_set_name(r, "r_add");
  2601. r = ggml_cpy(lora_ctx, r, dest_t);
  2602. offload_func(r);
  2603. ggml_set_name(r, "r_cpy");
  2604. }
  2605. struct ggml_cgraph gf = ggml_build_forward(r);
  2606. ggml_graph_compute_helper(work_buffer, &gf, n_threads);
  2607. // we won't need these tensors again, reset the context to save memory
  2608. ggml_free(lora_ctx);
  2609. lora_ctx = ggml_init(params);
  2610. lora_tensors.clear();
  2611. n_tensors++;
  2612. if (n_tensors % 4 == 0) {
  2613. fprintf(stderr, ".");
  2614. }
  2615. }
  2616. }
  2617. // TODO: this should be in a destructor, it will leak on failure
  2618. ggml_free(lora_ctx);
  2619. if (base_ctx) {
  2620. ggml_free(base_ctx);
  2621. }
  2622. const int64_t t_lora_us = ggml_time_us() - t_start_lora_us;
  2623. fprintf(stderr, " done (%.2f ms)\n", t_lora_us / 1000.0);
  2624. return 0;
  2625. }
  2626. int llama_apply_lora_from_file(struct llama_context * ctx, const char * path_lora, const char * path_base_model, int n_threads) {
  2627. try {
  2628. return llama_apply_lora_from_file_internal(ctx->model, path_lora, path_base_model, n_threads);
  2629. } catch (const std::exception & err) {
  2630. fprintf(stderr, "%s: failed to apply lora adapter: %s\n", __func__, err.what());
  2631. return 1;
  2632. }
  2633. }
  2634. int llama_model_apply_lora_from_file(const struct llama_model * model, const char * path_lora, const char * path_base_model, int n_threads) {
  2635. try {
  2636. return llama_apply_lora_from_file_internal(*model, path_lora, path_base_model, n_threads);
  2637. } catch (const std::exception & err) {
  2638. fprintf(stderr, "%s: failed to apply lora adapter: %s\n", __func__, err.what());
  2639. return 1;
  2640. }
  2641. }
  2642. int llama_get_kv_cache_token_count(const struct llama_context * ctx) {
  2643. return ctx->kv_self.n;
  2644. }
  2645. #define LLAMA_MAX_RNG_STATE (64*1024)
  2646. void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed) {
  2647. if (seed == LLAMA_DEFAULT_SEED) {
  2648. seed = time(NULL);
  2649. }
  2650. ctx->rng.seed(seed);
  2651. }
  2652. // Returns the *maximum* size of the state
  2653. size_t llama_get_state_size(const struct llama_context * ctx) {
  2654. // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state.
  2655. // for reference, std::mt19937(1337) serializes to 6701 bytes.
  2656. const size_t s_rng_size = sizeof(size_t);
  2657. const size_t s_rng = LLAMA_MAX_RNG_STATE;
  2658. const size_t s_logits_capacity = sizeof(size_t);
  2659. const size_t s_logits_size = sizeof(size_t);
  2660. const size_t s_logits = ctx->logits.capacity() * sizeof(float);
  2661. const size_t s_embedding_size = sizeof(size_t);
  2662. const size_t s_embedding = ctx->embedding.size() * sizeof(float);
  2663. const size_t s_kv_size = sizeof(size_t);
  2664. const size_t s_kv_ntok = sizeof(int);
  2665. const size_t s_kv = ctx->kv_self.buf.size;
  2666. const size_t s_total = (
  2667. + s_rng_size
  2668. + s_rng
  2669. + s_logits_capacity
  2670. + s_logits_size
  2671. + s_logits
  2672. + s_embedding_size
  2673. + s_embedding
  2674. + s_kv_size
  2675. + s_kv_ntok
  2676. + s_kv
  2677. );
  2678. return s_total;
  2679. }
  2680. // Copies the state to the specified destination address
  2681. size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) {
  2682. uint8_t * out = dst;
  2683. // copy rng
  2684. {
  2685. std::stringstream rng_ss;
  2686. rng_ss << ctx->rng;
  2687. const size_t rng_size = rng_ss.str().size();
  2688. char rng_buf[LLAMA_MAX_RNG_STATE];
  2689. memset(&rng_buf[0], 0, LLAMA_MAX_RNG_STATE);
  2690. memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());
  2691. memcpy(out, &rng_size, sizeof(rng_size)); out += sizeof(rng_size);
  2692. memcpy(out, &rng_buf[0], LLAMA_MAX_RNG_STATE); out += LLAMA_MAX_RNG_STATE;
  2693. }
  2694. // copy logits
  2695. {
  2696. const size_t logits_cap = ctx->logits.capacity();
  2697. const size_t logits_size = ctx->logits.size();
  2698. memcpy(out, &logits_cap, sizeof(logits_cap)); out += sizeof(logits_cap);
  2699. memcpy(out, &logits_size, sizeof(logits_size)); out += sizeof(logits_size);
  2700. if (logits_size) {
  2701. memcpy(out, ctx->logits.data(), logits_size * sizeof(float));
  2702. }
  2703. out += logits_cap * sizeof(float);
  2704. }
  2705. // copy embeddings
  2706. {
  2707. const size_t embedding_size = ctx->embedding.size();
  2708. memcpy(out, &embedding_size, sizeof(embedding_size)); out += sizeof(embedding_size);
  2709. if (embedding_size) {
  2710. memcpy(out, ctx->embedding.data(), embedding_size * sizeof(float));
  2711. out += embedding_size * sizeof(float);
  2712. }
  2713. }
  2714. // copy kv cache
  2715. {
  2716. const auto & kv_self = ctx->kv_self;
  2717. const auto & hparams = ctx->model.hparams;
  2718. const int n_layer = hparams.n_layer;
  2719. const int n_embd = hparams.n_embd;
  2720. const int n_ctx = hparams.n_ctx;
  2721. const size_t kv_size = kv_self.buf.size;
  2722. const int kv_ntok = llama_get_kv_cache_token_count(ctx);
  2723. memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size);
  2724. memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok);
  2725. if (kv_size) {
  2726. const size_t elt_size = ggml_element_size(kv_self.k);
  2727. ggml_context * cpy_ctx = ggml_init({ 4096, NULL, /* no_alloc */ true });
  2728. ggml_cgraph gf{};
  2729. ggml_tensor * kout3d = ggml_new_tensor_3d(cpy_ctx, kv_self.k->type, n_embd, kv_ntok, n_layer);
  2730. kout3d->data = out;
  2731. out += ggml_nbytes(kout3d);
  2732. ggml_tensor * vout3d = ggml_new_tensor_3d(cpy_ctx, kv_self.v->type, kv_ntok, n_embd, n_layer);
  2733. vout3d->data = out;
  2734. out += ggml_nbytes(vout3d);
  2735. ggml_tensor * k3d = ggml_view_3d(cpy_ctx, kv_self.k,
  2736. n_embd, kv_ntok, n_layer,
  2737. elt_size*n_embd, elt_size*n_embd*n_ctx, 0);
  2738. ggml_tensor * v3d = ggml_view_3d(cpy_ctx, kv_self.v,
  2739. kv_ntok, n_embd, n_layer,
  2740. elt_size*n_ctx, elt_size*n_ctx*n_embd, 0);
  2741. ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, k3d, kout3d));
  2742. ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, v3d, vout3d));
  2743. ggml_graph_compute_helper(ctx->work_buffer, &gf, /*n_threads*/ 1);
  2744. ggml_free(cpy_ctx);
  2745. }
  2746. }
  2747. const size_t written = out - dst;
  2748. const size_t max_size = llama_get_state_size(ctx);
  2749. LLAMA_ASSERT(written <= max_size);
  2750. return written;
  2751. }
  2752. // Sets the state reading from the specified source address
  2753. size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {
  2754. uint8_t * inp = src;
  2755. // set rng
  2756. {
  2757. size_t rng_size;
  2758. char rng_buf[LLAMA_MAX_RNG_STATE];
  2759. memcpy(&rng_size, inp, sizeof(rng_size)); inp += sizeof(rng_size);
  2760. memcpy(&rng_buf[0], inp, LLAMA_MAX_RNG_STATE); inp += LLAMA_MAX_RNG_STATE;
  2761. std::stringstream rng_ss;
  2762. rng_ss.str(std::string(&rng_buf[0], rng_size));
  2763. rng_ss >> ctx->rng;
  2764. LLAMA_ASSERT(rng_ss.fail() == false);
  2765. }
  2766. // set logits
  2767. {
  2768. size_t logits_cap;
  2769. size_t logits_size;
  2770. memcpy(&logits_cap, inp, sizeof(logits_cap)); inp += sizeof(logits_cap);
  2771. memcpy(&logits_size, inp, sizeof(logits_size)); inp += sizeof(logits_size);
  2772. LLAMA_ASSERT(ctx->logits.capacity() == logits_cap);
  2773. if (logits_size) {
  2774. ctx->logits.resize(logits_size);
  2775. memcpy(ctx->logits.data(), inp, logits_size * sizeof(float));
  2776. }
  2777. inp += logits_cap * sizeof(float);
  2778. }
  2779. // set embeddings
  2780. {
  2781. size_t embedding_size;
  2782. memcpy(&embedding_size, inp, sizeof(embedding_size)); inp += sizeof(embedding_size);
  2783. LLAMA_ASSERT(ctx->embedding.capacity() == embedding_size);
  2784. if (embedding_size) {
  2785. memcpy(ctx->embedding.data(), inp, embedding_size * sizeof(float));
  2786. inp += embedding_size * sizeof(float);
  2787. }
  2788. }
  2789. // set kv cache
  2790. {
  2791. const auto & kv_self = ctx->kv_self;
  2792. const auto & hparams = ctx->model.hparams;
  2793. const int n_layer = hparams.n_layer;
  2794. const int n_embd = hparams.n_embd;
  2795. const int n_ctx = hparams.n_ctx;
  2796. size_t kv_size;
  2797. int kv_ntok;
  2798. memcpy(&kv_size, inp, sizeof(kv_size)); inp += sizeof(kv_size);
  2799. memcpy(&kv_ntok, inp, sizeof(kv_ntok)); inp += sizeof(kv_ntok);
  2800. if (kv_size) {
  2801. LLAMA_ASSERT(kv_self.buf.size == kv_size);
  2802. const size_t elt_size = ggml_element_size(kv_self.k);
  2803. ggml_context * cpy_ctx = ggml_init({ 4096, NULL, /* no_alloc */ true });
  2804. ggml_cgraph gf{};
  2805. ggml_tensor * kin3d = ggml_new_tensor_3d(cpy_ctx, kv_self.k->type, n_embd, kv_ntok, n_layer);
  2806. kin3d->data = (void *) inp;
  2807. inp += ggml_nbytes(kin3d);
  2808. ggml_tensor * vin3d = ggml_new_tensor_3d(cpy_ctx, kv_self.v->type, kv_ntok, n_embd, n_layer);
  2809. vin3d->data = (void *) inp;
  2810. inp += ggml_nbytes(vin3d);
  2811. ggml_tensor * k3d = ggml_view_3d(cpy_ctx, kv_self.k,
  2812. n_embd, kv_ntok, n_layer,
  2813. elt_size*n_embd, elt_size*n_embd*n_ctx, 0);
  2814. ggml_tensor * v3d = ggml_view_3d(cpy_ctx, kv_self.v,
  2815. kv_ntok, n_embd, n_layer,
  2816. elt_size*n_ctx, elt_size*n_ctx*n_embd, 0);
  2817. ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, kin3d, k3d));
  2818. ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, vin3d, v3d));
  2819. ggml_graph_compute_helper(ctx->work_buffer, &gf, /*n_threads*/ 1);
  2820. ggml_free(cpy_ctx);
  2821. }
  2822. ctx->kv_self.n = kv_ntok;
  2823. }
  2824. const size_t nread = inp - src;
  2825. const size_t max_size = llama_get_state_size(ctx);
  2826. LLAMA_ASSERT(nread <= max_size);
  2827. return nread;
  2828. }
  2829. static bool llama_load_session_file_internal(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
  2830. llama_file file(path_session, "rb");
  2831. // sanity checks
  2832. {
  2833. const uint32_t magic = file.read_u32();
  2834. const uint32_t version = file.read_u32();
  2835. if (magic != LLAMA_SESSION_MAGIC || version != LLAMA_SESSION_VERSION) {
  2836. fprintf(stderr, "%s : unknown (magic, version) for session file: %08x, %08x\n", __func__, magic, version);
  2837. return false;
  2838. }
  2839. llama_hparams session_hparams;
  2840. file.read_raw(&session_hparams, sizeof(llama_hparams));
  2841. if (session_hparams != ctx->model.hparams) {
  2842. fprintf(stderr, "%s : model hparams didn't match from session file!\n", __func__);
  2843. return false;
  2844. }
  2845. }
  2846. // load the prompt
  2847. {
  2848. const uint32_t n_token_count = file.read_u32();
  2849. if (n_token_count > n_token_capacity) {
  2850. fprintf(stderr, "%s : token count in session file exceeded capacity! %u > %zu\n", __func__, n_token_count, n_token_capacity);
  2851. return false;
  2852. }
  2853. file.read_raw(tokens_out, sizeof(llama_token) * n_token_count);
  2854. *n_token_count_out = n_token_count;
  2855. }
  2856. // restore the context state
  2857. {
  2858. const size_t n_state_size_cur = file.size - file.tell();
  2859. const size_t n_state_size_max = llama_get_state_size(ctx);
  2860. if (n_state_size_cur > n_state_size_max) {
  2861. fprintf(stderr, "%s : the state size in session file is too big! max %zu, got %zu\n", __func__, n_state_size_max, n_state_size_cur);
  2862. return false;
  2863. }
  2864. std::vector<uint8_t> state_data(n_state_size_max);
  2865. file.read_raw(state_data.data(), n_state_size_cur);
  2866. llama_set_state_data(ctx, state_data.data());
  2867. }
  2868. return true;
  2869. }
  2870. bool llama_load_session_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
  2871. try {
  2872. return llama_load_session_file_internal(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out);
  2873. } catch (const std::exception & err) {
  2874. fprintf(stderr, "error loading session file: %s\n", err.what());
  2875. return false;
  2876. }
  2877. }
  2878. bool llama_save_session_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) {
  2879. llama_file file(path_session, "wb");
  2880. file.write_u32(LLAMA_SESSION_MAGIC);
  2881. file.write_u32(LLAMA_SESSION_VERSION);
  2882. file.write_raw(&ctx->model.hparams, sizeof(llama_hparams));
  2883. // save the prompt
  2884. file.write_u32((uint32_t) n_token_count);
  2885. file.write_raw(tokens, sizeof(llama_token) * n_token_count);
  2886. // save the context state
  2887. {
  2888. const size_t n_state_size_max = llama_get_state_size(ctx);
  2889. std::vector<uint8_t> state_data(n_state_size_max);
  2890. const size_t n_state_size_cur = llama_copy_state_data(ctx, state_data.data());
  2891. file.write_raw(state_data.data(), n_state_size_cur);
  2892. }
  2893. return true;
  2894. }
  2895. int llama_eval(
  2896. struct llama_context * ctx,
  2897. const llama_token * tokens,
  2898. int n_tokens,
  2899. int n_past,
  2900. int n_threads) {
  2901. if (!llama_eval_internal(*ctx, tokens, nullptr, n_tokens, n_past, n_threads, nullptr)) {
  2902. fprintf(stderr, "%s: failed to eval\n", __func__);
  2903. return 1;
  2904. }
  2905. // get a more accurate load time, upon first eval
  2906. // TODO: fix this
  2907. if (!ctx->has_evaluated_once) {
  2908. ctx->t_load_us = ggml_time_us() - ctx->t_start_us;
  2909. ctx->has_evaluated_once = true;
  2910. }
  2911. return 0;
  2912. }
  2913. int llama_eval_embd(
  2914. struct llama_context * ctx,
  2915. const float * embd,
  2916. int n_tokens,
  2917. int n_past,
  2918. int n_threads) {
  2919. if (!llama_eval_internal(*ctx, nullptr, embd, n_tokens, n_past, n_threads, nullptr)) {
  2920. fprintf(stderr, "%s: failed to eval\n", __func__);
  2921. return 1;
  2922. }
  2923. // get a more accurate load time, upon first eval
  2924. // TODO: fix this
  2925. if (!ctx->has_evaluated_once) {
  2926. ctx->t_load_us = ggml_time_us() - ctx->t_start_us;
  2927. ctx->has_evaluated_once = true;
  2928. }
  2929. return 0;
  2930. }
  2931. int llama_eval_export(struct llama_context * ctx, const char * fname) {
  2932. const int n_batch = 1;
  2933. const int n_ctx = 512 - n_batch;
  2934. const std::vector<llama_token> tmp(n_batch, llama_token_bos());
  2935. if (!llama_eval_internal(*ctx, tmp.data(), nullptr, tmp.size(), n_ctx, 1, fname)) {
  2936. fprintf(stderr, "%s: failed to eval\n", __func__);
  2937. return 1;
  2938. }
  2939. return 0;
  2940. }
  2941. int llama_tokenize(
  2942. struct llama_context * ctx,
  2943. const char * text,
  2944. llama_token * tokens,
  2945. int n_max_tokens,
  2946. bool add_bos) {
  2947. auto res = llama_tokenize(ctx->vocab, text, add_bos);
  2948. if (n_max_tokens < (int) res.size()) {
  2949. fprintf(stderr, "%s: too many tokens\n", __func__);
  2950. return -((int) res.size());
  2951. }
  2952. for (size_t i = 0; i < res.size(); i++) {
  2953. tokens[i] = res[i];
  2954. }
  2955. return res.size();
  2956. }
  2957. int llama_n_vocab(const struct llama_context * ctx) {
  2958. return ctx->vocab.id_to_token.size();
  2959. }
  2960. int llama_n_ctx(const struct llama_context * ctx) {
  2961. return ctx->model.hparams.n_ctx;
  2962. }
  2963. int llama_n_embd(const struct llama_context * ctx) {
  2964. return ctx->model.hparams.n_embd;
  2965. }
  2966. int llama_get_vocab(
  2967. const struct llama_context * ctx,
  2968. const char * * strings,
  2969. float * scores,
  2970. int capacity) {
  2971. int n = std::min(capacity, (int) ctx->vocab.id_to_token.size());
  2972. for (int i = 0; i<n; ++i) {
  2973. strings[i] = ctx->vocab.id_to_token[i].tok.c_str();
  2974. scores[i] = ctx->vocab.id_to_token[i].score;
  2975. }
  2976. return n;
  2977. }
  2978. float * llama_get_logits(struct llama_context * ctx) {
  2979. return ctx->logits.data();
  2980. }
  2981. float * llama_get_embeddings(struct llama_context * ctx) {
  2982. return ctx->embedding.data();
  2983. }
  2984. const char * llama_token_to_str(const struct llama_context * ctx, llama_token token) {
  2985. if (token >= llama_n_vocab(ctx)) {
  2986. return nullptr;
  2987. }
  2988. return ctx->vocab.id_to_token[token].tok.c_str();
  2989. }
  2990. llama_token llama_token_bos() {
  2991. return 1;
  2992. }
  2993. llama_token llama_token_eos() {
  2994. return 2;
  2995. }
  2996. llama_token llama_token_nl() {
  2997. return 13;
  2998. }
  2999. struct llama_timings llama_get_timings(struct llama_context * ctx) {
  3000. struct llama_timings result = {
  3001. /*.t_start_ms =*/ 1e-3 * ctx->t_start_us,
  3002. /*.t_end_ms =*/ 1.00 * ggml_time_ms(),
  3003. /*.t_load_ms =*/ 1e-3 * ctx->t_load_us,
  3004. /*.t_sample_ms =*/ 1e-3 * ctx->t_sample_us,
  3005. /*.t_p_eval_ms =*/ 1e-3 * ctx->t_p_eval_us,
  3006. /*.t_eval_ms =*/ 1e-3 * ctx->t_eval_us,
  3007. /*.n_sample =*/ std::max(1, ctx->n_sample),
  3008. /*.n_p_eval =*/ std::max(1, ctx->n_p_eval),
  3009. /*.n_eval =*/ std::max(1, ctx->n_eval),
  3010. };
  3011. return result;
  3012. }
  3013. void llama_print_timings(struct llama_context * ctx) {
  3014. const llama_timings timings = llama_get_timings(ctx);
  3015. fprintf(stderr, "\n");
  3016. fprintf(stderr, "%s: load time = %8.2f ms\n", __func__, timings.t_load_ms);
  3017. fprintf(stderr, "%s: sample time = %8.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n",
  3018. __func__, timings.t_sample_ms, timings.n_sample, timings.t_sample_ms / timings.n_sample, 1e3 / timings.t_sample_ms * timings.n_sample);
  3019. fprintf(stderr, "%s: prompt eval time = %8.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n",
  3020. __func__, timings.t_p_eval_ms, timings.n_p_eval, timings.t_p_eval_ms / timings.n_p_eval, 1e3 / timings.t_p_eval_ms * timings.n_p_eval);
  3021. fprintf(stderr, "%s: eval time = %8.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n",
  3022. __func__, timings.t_eval_ms, timings.n_eval, timings.t_eval_ms / timings.n_eval, 1e3 / timings.t_eval_ms * timings.n_eval);
  3023. fprintf(stderr, "%s: total time = %8.2f ms\n", __func__, (timings.t_end_ms - timings.t_start_ms));
  3024. }
  3025. void llama_reset_timings(struct llama_context * ctx) {
  3026. ctx->t_start_us = ggml_time_us();
  3027. ctx->t_sample_us = ctx->n_sample = 0;
  3028. ctx->t_eval_us = ctx->n_eval = 0;
  3029. ctx->t_p_eval_us = ctx->n_p_eval = 0;
  3030. }
  3031. const char * llama_print_system_info(void) {
  3032. static std::string s;
  3033. s = "";
  3034. s += "AVX = " + std::to_string(ggml_cpu_has_avx()) + " | ";
  3035. s += "AVX2 = " + std::to_string(ggml_cpu_has_avx2()) + " | ";
  3036. s += "AVX512 = " + std::to_string(ggml_cpu_has_avx512()) + " | ";
  3037. s += "AVX512_VBMI = " + std::to_string(ggml_cpu_has_avx512_vbmi()) + " | ";
  3038. s += "AVX512_VNNI = " + std::to_string(ggml_cpu_has_avx512_vnni()) + " | ";
  3039. s += "FMA = " + std::to_string(ggml_cpu_has_fma()) + " | ";
  3040. s += "NEON = " + std::to_string(ggml_cpu_has_neon()) + " | ";
  3041. s += "ARM_FMA = " + std::to_string(ggml_cpu_has_arm_fma()) + " | ";
  3042. s += "F16C = " + std::to_string(ggml_cpu_has_f16c()) + " | ";
  3043. s += "FP16_VA = " + std::to_string(ggml_cpu_has_fp16_va()) + " | ";
  3044. s += "WASM_SIMD = " + std::to_string(ggml_cpu_has_wasm_simd()) + " | ";
  3045. s += "BLAS = " + std::to_string(ggml_cpu_has_blas()) + " | ";
  3046. s += "SSE3 = " + std::to_string(ggml_cpu_has_sse3()) + " | ";
  3047. s += "VSX = " + std::to_string(ggml_cpu_has_vsx()) + " | ";
  3048. return s.c_str();
  3049. }
  3050. // For internal test use
  3051. const std::vector<std::pair<std::string, struct ggml_tensor *>>& llama_internal_get_tensor_map(struct llama_context * ctx) {
  3052. return ctx->model.tensors_by_name;
  3053. }