diff --git a/Cargo.lock b/Cargo.lock index aa9e891..5c46167 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.0" @@ -29,6 +38,17 @@ version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +[[package]] +name = "async-trait" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "auditable-serde" version = "0.8.0" @@ -47,18 +67,367 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-config" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fcc63c9860579e4cb396239570e979376e70aab79e496621748a09913f8b36" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http 0.62.1", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 1.3.1", + "time", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "687bc16bc431a8533fe0097c7f0182874767f920989d7260950172ae8e3c4465" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c4063282c69991e57faab9e5cb21ae557e59f5b0fb285c196335243df8dc25c" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http 0.62.1", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-bedrockruntime" +version = "1.86.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db14a0566037a6c686ef075c406dec4b067537af3d76950522e9e89848ce7a5a" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http 0.62.1", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "hyper", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.67.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74e8e9ac4a837859c8f1d747054172e1e55933f02ed34728b0b34dea0591ec84" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.62.1", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3503af839bd8751d0bdc5a46b9cac93a003a353e635b0c12cf2376b5b53e41ea" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http 0.62.1", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "percent-encoding", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c45d3dddac16c5c59d553ece225a88870cf81b7b813c9cc17b78cf4685eac7a" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http" +version = "0.62.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99335bec6cdc50a346fda1437f9fefe33abf8c99060739a546a16457f2862ca9" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92144e45819cae7dc62af23eac5a038a58aa544432d2102609654376a900bd07" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14302f06d1d5b7d333fd819943075b13d27c7700b414f574c3c35859bfb55d5e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http 0.62.1", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e5d9e3a80a18afa109391fb5ad09c3daf887b516c6fd805a157c6ea7994a57" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.3.1", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40076bd09fadbc12d5e026ae080d0930defa606856186e31d83ccc6a255eeaf3" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a322fec39e4df22777ed3ad8ea868ac2f94cd15e1a55f6ee8d8d6305057689a" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "bitflags" version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.17.0" @@ -71,6 +440,16 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "camino" version = "1.1.9" @@ -139,6 +518,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -148,6 +536,36 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -159,6 +577,12 @@ dependencies = [ "syn", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -174,6 +598,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "flate2" version = "1.1.1" @@ -294,6 +724,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.3.2" @@ -306,6 +746,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + [[package]] name = "git-version" version = "0.3.9" @@ -353,6 +799,26 @@ dependencies = [ "wit-bindgen-rt 0.40.0", ] +[[package]] +name = "golem-llm-bedrock" +version = "0.0.0" +dependencies = [ + "async-trait", + "aws-config", + "aws-sdk-bedrockruntime", + "aws-smithy-http 0.60.12", + "aws-smithy-runtime", + "aws-types", + "bytes", + "futures-util", + "golem-llm", + "golem-rust", + "log", + "serde", + "serde_json", + "wit-bindgen-rt 0.40.0", +] + [[package]] name = "golem-llm-grok" version = "0.0.0" @@ -455,6 +921,32 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.3.1" @@ -466,6 +958,74 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.3.1", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "iana-time-zone" version = "0.1.63" @@ -729,6 +1289,21 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -738,12 +1313,27 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -762,6 +1352,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "prettyplease" version = "0.2.32" @@ -796,6 +1392,12 @@ version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "reqwest" version = "0.12.15" @@ -806,7 +1408,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "http", + "http 1.3.1", "mime", "percent-encoding", "serde", @@ -818,6 +1420,21 @@ dependencies = [ "wit-bindgen-rt 0.41.0", ] +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustversion" version = "1.0.20" @@ -889,6 +1506,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -925,6 +1553,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.100" @@ -976,6 +1610,36 @@ dependencies = [ "syn", ] +[[package]] +name = "time" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + +[[package]] +name = "time-macros" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -986,6 +1650,30 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tokio" +version = "1.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" +dependencies = [ + "backtrace", + "bytes", + "pin-project-lite", +] + +[[package]] +name = "tokio-util" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "topological-sort" version = "0.2.2" @@ -998,6 +1686,49 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + [[package]] name = "unicode-ident" version = "1.0.18" @@ -1027,6 +1758,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf16_iter" version = "1.0.5" @@ -1050,6 +1787,27 @@ dependencies = [ "sha1_smol", ] +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.14.2+wasi-0.2.4" @@ -1263,6 +2021,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wit-bindgen" version = "0.24.0" @@ -1487,6 +2309,12 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yoke" version = "0.7.5" @@ -1532,6 +2360,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + [[package]] name = "zerovec" version = "0.10.4" diff --git a/Cargo.toml b/Cargo.toml index c33513d..1e24f00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] resolver = "2" -members = ["llm", "llm-anthropic", "llm-grok", "llm-openai", "llm-openrouter"] +members = ["llm", "llm-anthropic", "llm-grok", "llm-openai", "llm-openrouter", "llm-bedrock"] [profile.release] debug = false diff --git a/Makefile.toml b/Makefile.toml index 8b7f96f..384a20d 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -50,12 +50,23 @@ install_crate = { crate_name = "cargo-component", version = "0.20.0" } command = "cargo-component" args = ["build", "-p", "golem-llm-openrouter", "--no-default-features"] +[tasks.build-bedrock] +install_crate = { crate_name = "cargo-component", version = "0.20.0" } +command = "cargo-component" +args = ["build", "-p", "golem-llm-bedrock"] + +[tasks.build-bedrock-portable] +install_crate = { crate_name = "cargo-component", version = "0.20.0" } +command = "cargo-component" +args = ["build", "-p", "golem-llm-bedrock", "--no-default-features"] + [tasks.build] dependencies = [ "build-anthropic", "build-grok", "build-openai", "build-openrouter", + "build-bedrock", ] [tasks.build-portable] @@ -64,6 +75,7 @@ dependencies = [ "build-grok-portable", "build-openai-portable", "build-openrouter-portable", + "build-bedrock-portable", ] [tasks.build-all] @@ -78,6 +90,7 @@ cp target/wasm32-wasip1/debug/golem_llm_anthropic.wasm components/debug/golem_ll cp target/wasm32-wasip1/debug/golem_llm_grok.wasm components/debug/golem_llm_grok.wasm cp target/wasm32-wasip1/debug/golem_llm_openai.wasm components/debug/golem_llm_openai.wasm cp target/wasm32-wasip1/debug/golem_llm_openrouter.wasm components/debug/golem_llm_openrouter.wasm +cp target/wasm32-wasip1/debug/golem_llm_bedrock.wasm components/debug/golem_llm_bedrock.wasm cm_run_task clean cm_run_task build-portable @@ -86,6 +99,7 @@ cp target/wasm32-wasip1/debug/golem_llm_anthropic.wasm components/debug/golem_ll cp target/wasm32-wasip1/debug/golem_llm_grok.wasm components/debug/golem_llm_grok-portable.wasm cp target/wasm32-wasip1/debug/golem_llm_openai.wasm components/debug/golem_llm_openai-portable.wasm cp target/wasm32-wasip1/debug/golem_llm_openrouter.wasm components/debug/golem_llm_openrouter-portable.wasm +cp target/wasm32-wasip1/debug/golem_llm_bedrock.wasm components/debug/golem_llm_bedrock-portable.wasm ''' [tasks.release-build-anthropic] @@ -140,12 +154,29 @@ args = [ "--no-default-features", ] +[tasks.release-build-bedrock] +install_crate = { crate_name = "cargo-component", version = "0.20.0" } +command = "cargo-component" +args = ["build", "-p", "golem-llm-bedrock", "--release"] + +[tasks.release-build-bedrock-portable] +install_crate = { crate_name = "cargo-component", version = "0.20.0" } +command = "cargo-component" +args = [ + "build", + "-p", + "golem-llm-bedrock", + "--release", + "--no-default-features", +] + [tasks.release-build] dependencies = [ "release-build-anthropic", "release-build-grok", "release-build-openai", "release-build-openrouter", + "release-build-bedrock", ] [tasks.release-build-portable] @@ -154,6 +185,7 @@ dependencies = [ "release-build-grok-portable", "release-build-openai-portable", "release-build-openrouter-portable", + "release-build-bedrock-portable", ] [tasks.release-build-all] @@ -170,6 +202,7 @@ cp target/wasm32-wasip1/release/golem_llm_anthropic.wasm components/release/gole cp target/wasm32-wasip1/release/golem_llm_grok.wasm components/release/golem_llm_grok.wasm cp target/wasm32-wasip1/release/golem_llm_openai.wasm components/release/golem_llm_openai.wasm cp target/wasm32-wasip1/release/golem_llm_openrouter.wasm components/release/golem_llm_openrouter.wasm +cp target/wasm32-wasip1/release/golem_llm_bedrock.wasm components/release/golem_llm_bedrock.wasm cm_run_task clean cm_run_task release-build-portable @@ -178,6 +211,7 @@ cp target/wasm32-wasip1/release/golem_llm_anthropic.wasm components/release/gole cp target/wasm32-wasip1/release/golem_llm_grok.wasm components/release/golem_llm_grok-portable.wasm cp target/wasm32-wasip1/release/golem_llm_openai.wasm components/release/golem_llm_openai-portable.wasm cp target/wasm32-wasip1/release/golem_llm_openrouter.wasm components/release/golem_llm_openrouter-portable.wasm +cp target/wasm32-wasip1/release/golem_llm_bedrock.wasm components/release/golem_llm_bedrock-portable.wasm ''' [tasks.wit-update] @@ -197,6 +231,7 @@ dependencies = ["wit-update"] # "llm-grok/wit/deps/golem-llm/golem-llm.wit", # "llm-openai/wit/deps/golem-llm/golem-llm.wit", # "llm-openrouter/wit/deps/golem-llm/golem-llm.wit", +# "llm-bedrock/wit/deps/golem-llm/golem-llm.wit", #] } } script_runner = "@duckscript" @@ -221,6 +256,10 @@ rm -r llm-openrouter/wit/deps mkdir llm-openrouter/wit/deps/golem-llm cp wit/golem-llm.wit llm-openrouter/wit/deps/golem-llm/golem-llm.wit cp wit/deps/wasi:io llm-openrouter/wit/deps +rm -r llm-bedrock/wit/deps +mkdir llm-bedrock/wit/deps/golem-llm +cp wit/golem-llm.wit llm-bedrock/wit/deps/golem-llm/golem-llm.wit +cp wit/deps/wasi:io llm-bedrock/wit/deps rm -r test/wit mkdir test/wit/deps/golem-llm diff --git a/README.md b/README.md index 4b921a3..87b9197 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,12 @@ There are 8 published WASM files for each release: | `golem-llm-grok.wasm` | LLM implementation for xAI (Grok), using custom Golem specific durability features | | `golem-llm-openai.wasm` | LLM implementation for OpenAI, using custom Golem specific durability features | | `golem-llm-openrouter.wasm` | LLM implementation for OpenRouter, using custom Golem specific durability features | +| `golem-llm-bedrock.wasm` | LLM implementation for AWS Bedrock, using custom Golem specific durability features | | `golem-llm-anthropic-portable.wasm` | LLM implementation for Anthropic AI, with no Golem specific dependencies. | | `golem-llm-grok-portable.wasm` | LLM implementation for xAI (Grok), with no Golem specific dependencies. | | `golem-llm-openai-portable.wasm` | LLM implementation for OpenAI, with no Golem specific dependencies. | | `golem-llm-openrouter-portable.wasm` | LLM implementation for OpenRouter, with no Golem specific dependencies. | +| `golem-llm-bedrock-portable.wasm` | LLM implementation for AWS Bedrock, with no Golem specific dependencies. | Every component **exports** the same `golem:llm` interface, [defined here](wit/golem-llm.wit). @@ -34,6 +36,7 @@ Each provider has to be configured with an API key passed as an environment vari | Grok | `XAI_API_KEY` | | OpenAI | `OPENAI_API_KEY` | | OpenRouter | `OPENROUTER_API_KEY` | +| Bedrock | AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN (optional), AWS_REGION (or AWS_DEFAULT_REGION). Relies on standard AWS SDK credential chain. The region can also be set via provider-options in the Config with key AWS_REGION. | Additionally, setting the `GOLEM_LLM_LOG=trace` environment variable enables trace logging for all the communication with the underlying LLM provider. diff --git a/llm-bedrock/Cargo.toml b/llm-bedrock/Cargo.toml new file mode 100644 index 0000000..2d065ca --- /dev/null +++ b/llm-bedrock/Cargo.toml @@ -0,0 +1,55 @@ +[package] +name = "golem-llm-bedrock" +version = "0.0.0" +edition = "2021" +license = "Apache-2.0" +homepage = "https://golem.cloud" +repository = "https://github.com/golemcloud/golem-llm" +description = "WebAssembly component for working with AWS Bedrock APIs, with special support for Golem Cloud" + +[lib] +path = "src/lib.rs" +crate-type = ["cdylib"] + +[features] +default = ["durability"] +durability = ["golem-rust/durability", "golem-llm/durability"] + +[dependencies] +golem-llm = { path = "../llm", version = "0.0.0", default-features = false } +golem-rust = { workspace = true } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +wit-bindgen-rt = { workspace = true } + +# AWS SDK +aws-config = { version = "1.1.7", default-features = false, features = ["behavior-version-latest"] } +aws-sdk-bedrockruntime = { version = "1.16.0", default-features = false, features = ["rt-tokio"] } +aws-smithy-http = { version = "0.60.0" } +aws-smithy-runtime = { version = "1.1.7", features = ["client"] } +aws-types = { version = "1.1.1", default-features = false } # For Region + +# Async utilities +futures-util = { version = "0.3", default-features = false, features = ["alloc"] } +async-trait = "0.1" # If needed for traits with async methods + +# For Bedrock JSON structures and byte manipulation +bytes = "1.5" + +[package.metadata.component] +package = "golem:llm-bedrock" + +[package.metadata.component.bindings] +generate_unused_types = true + +[package.metadata.component.bindings.with] +"golem:llm/llm@1.0.0" = "golem_llm::golem::llm::llm" +"wasi:io/poll@0.2.0" = "golem_rust::wasm_rpc::wasi::io::poll" + +[package.metadata.component.target] +path = "wit" + +[package.metadata.component.target.dependencies] +"golem:llm" = { path = "wit/deps/golem-llm" } +"wasi:io" = { path = "wit/deps/wasi:io"} \ No newline at end of file diff --git a/llm-bedrock/src/client.rs b/llm-bedrock/src/client.rs new file mode 100644 index 0000000..bec116c --- /dev/null +++ b/llm-bedrock/src/client.rs @@ -0,0 +1,175 @@ +use aws_config::meta::region::RegionProviderChain; +use aws_sdk_bedrockruntime::primitives::Blob; +use aws_sdk_bedrockruntime::Client as AwsBedrockClient; +use aws_sdk_bedrockruntime::error::ProvideErrorMetadata; +use aws_types::SdkConfig; +use golem_llm::golem::llm::llm::{Error, ErrorCode}; +use log::{debug, error, trace}; + +pub struct BedrockClient { + client: AwsBedrockClient, +} + +// Helper to construct SDK config using SmithyWasmClient +async fn new_bedrock_sdk_config(aws_region_opt: Option) -> Result { + let region_provider = RegionProviderChain::first_try(aws_region_opt.map(aws_types::region::Region::new)) + .or_default_provider() + .or_else(aws_types::region::Region::new("us-east-1")); // Default fallback region + + let sdk_config = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .region(region_provider) + .load() + .await; + Ok(sdk_config) +} + +impl BedrockClient { + pub async fn new(aws_region_from_config: Option) -> Result { + debug!( + "Initializing BedrockClient with region from config: {:?}", + aws_region_from_config + ); + match new_bedrock_sdk_config(aws_region_from_config).await { + Ok(sdk_config) => { + let client = AwsBedrockClient::new(&sdk_config); + Ok(Self { client }) + } + Err(e) => { + error!("Failed to initialize Bedrock SDK config: {}", e.message); + Err(e) + } + } + } + + pub async fn invoke_model( + &self, + model_id: String, + body: serde_json::Value, + accept: String, + content_type: String, + ) -> Result { + trace!( + "Invoking Bedrock model. Model ID: {}, Content-Type: {}, Accept: {}, Body: {}", + model_id, content_type, accept, body + ); + + let body_blob = Blob::new(body.to_string()); + + let response = self + .client + .invoke_model() + .model_id(model_id) + .body(body_blob) + .content_type(content_type) + .accept(accept) + .send() + .await + .map_err(|sdk_err| { + let error_message = format!("Bedrock InvokeModel SDK error: {:?}", sdk_err); + error!("{}", error_message); + let provider_error_json = Some(error_message.clone()); + let message = sdk_err.message().unwrap_or("Unknown Bedrock SDK error").to_string(); + + let code = match sdk_err.as_service_error() { + Some(err) => match err { + aws_sdk_bedrockruntime::operation::invoke_model::InvokeModelError::ValidationException(_) => ErrorCode::InvalidRequest, + aws_sdk_bedrockruntime::operation::invoke_model::InvokeModelError::AccessDeniedException(_) => ErrorCode::AuthenticationFailed, + aws_sdk_bedrockruntime::operation::invoke_model::InvokeModelError::ResourceNotFoundException(_) => ErrorCode::InvalidRequest, + aws_sdk_bedrockruntime::operation::invoke_model::InvokeModelError::ThrottlingException(_) => ErrorCode::RateLimitExceeded, + aws_sdk_bedrockruntime::operation::invoke_model::InvokeModelError::ServiceQuotaExceededException(_) => ErrorCode::RateLimitExceeded, + aws_sdk_bedrockruntime::operation::invoke_model::InvokeModelError::ModelTimeoutException(_) => ErrorCode::InternalError, + aws_sdk_bedrockruntime::operation::invoke_model::InvokeModelError::InternalServerException(_) => ErrorCode::InternalError, + aws_sdk_bedrockruntime::operation::invoke_model::InvokeModelError::ModelNotReadyException(_) => ErrorCode::Unsupported, + aws_sdk_bedrockruntime::operation::invoke_model::InvokeModelError::ModelErrorException(_) => ErrorCode::InternalError, + _ => ErrorCode::InternalError, + }, + None => ErrorCode::InternalError, + }; + Error { + code, + message, + provider_error_json, + } + })?; + + let output_body_bytes = response.body.into_inner(); + let output_body_str = std::str::from_utf8(&output_body_bytes).map_err(|e| { + let msg = format!("Failed to parse Bedrock response body as UTF-8: {e}"); + error!("{}", msg); + Error { + code: ErrorCode::InternalError, + message: msg, + provider_error_json: None, + } + })?; + + trace!("Received Bedrock response body: {}", output_body_str); + + serde_json::from_str(output_body_str).map_err(|e| { + let msg = format!("Failed to parse Bedrock response JSON: {e}"); + error!("{} Body was: {}", msg, output_body_str); + Error { + code: ErrorCode::InternalError, + message: msg, + provider_error_json: Some(output_body_str.to_string()), + } + }) + } + + pub async fn invoke_model_with_response_stream( + &self, + model_id: String, + body: serde_json::Value, + accept: String, + content_type: String, + ) -> Result< + aws_sdk_bedrockruntime::operation::invoke_model_with_response_stream::InvokeModelWithResponseStreamOutput, + Error + > { + trace!( + "Invoking Bedrock model with response stream. Model ID: {}, Body: {}", + model_id, body + ); + let body_blob = Blob::new(body.to_string()); + + self.client + .invoke_model_with_response_stream() + .model_id(model_id) + .body(body_blob) + .content_type(content_type) + .accept(accept) + .send() + .await + .map_err(|sdk_err| { + let error_message = format!( + "Bedrock InvokeModelWithResponseStream SDK error: {:?}", + sdk_err + ); + error!("{}", error_message); + let provider_error_json = Some(error_message.clone()); + let message = sdk_err.message().unwrap_or("Unknown Bedrock SDK error for stream").to_string(); + + let code = match sdk_err.as_service_error() { + Some(err) => match err { + aws_sdk_bedrockruntime::operation::invoke_model_with_response_stream::InvokeModelWithResponseStreamError::ValidationException(_) => ErrorCode::InvalidRequest, + aws_sdk_bedrockruntime::operation::invoke_model_with_response_stream::InvokeModelWithResponseStreamError::AccessDeniedException(_) => ErrorCode::AuthenticationFailed, + aws_sdk_bedrockruntime::operation::invoke_model_with_response_stream::InvokeModelWithResponseStreamError::ResourceNotFoundException(_) => ErrorCode::InvalidRequest, + aws_sdk_bedrockruntime::operation::invoke_model_with_response_stream::InvokeModelWithResponseStreamError::ThrottlingException(_) => ErrorCode::RateLimitExceeded, + aws_sdk_bedrockruntime::operation::invoke_model_with_response_stream::InvokeModelWithResponseStreamError::ServiceQuotaExceededException(_) => ErrorCode::RateLimitExceeded, + aws_sdk_bedrockruntime::operation::invoke_model_with_response_stream::InvokeModelWithResponseStreamError::ModelTimeoutException(_) => ErrorCode::InternalError, + aws_sdk_bedrockruntime::operation::invoke_model_with_response_stream::InvokeModelWithResponseStreamError::InternalServerException(_) => ErrorCode::InternalError, + aws_sdk_bedrockruntime::operation::invoke_model_with_response_stream::InvokeModelWithResponseStreamError::ModelNotReadyException(_) => ErrorCode::Unsupported, + aws_sdk_bedrockruntime::operation::invoke_model_with_response_stream::InvokeModelWithResponseStreamError::ModelErrorException(_) => ErrorCode::InternalError, + aws_sdk_bedrockruntime::operation::invoke_model_with_response_stream::InvokeModelWithResponseStreamError::ModelStreamErrorException(_) => ErrorCode::InternalError, + _ => ErrorCode::InternalError, + }, + None => ErrorCode::InternalError, + }; + Error { + code, + message, + provider_error_json, + } + }) + } +} \ No newline at end of file diff --git a/llm-bedrock/src/conversions.rs b/llm-bedrock/src/conversions.rs new file mode 100644 index 0000000..787e3b2 --- /dev/null +++ b/llm-bedrock/src/conversions.rs @@ -0,0 +1,625 @@ +use golem_llm::golem::llm::llm::{ + ChatEvent, CompleteResponse, ContentPart, Error, ErrorCode, FinishReason, Message, + ResponseMetadata, Role, StreamDelta, StreamEvent, ToolCall, ToolDefinition, ToolResult, Usage, Config +}; +use log::{warn, trace}; +use serde_json::json; +use std::collections::HashMap; + +// --- Request Conversion --- + +// Helper to determine Bedrock model family from model_id string +fn get_bedrock_model_family(model_id: &str) -> &str { + if model_id.starts_with("anthropic.claude") { + "claude" + } else if model_id.starts_with("ai21.j2") { + "jurassic2" + } else if model_id.starts_with("amazon.titan-text") { + "titan-text" + } else if model_id.starts_with("cohere.command") { + "cohere-command" + } else if model_id.starts_with("meta.llama2") || model_id.starts_with("meta.llama3") { + "llama" + } + // Add more model families as needed + else { + warn!("Unknown Bedrock model family for model_id: {}. Defaulting to generic or expecting provider_options.", model_id); + "unknown" // Or handle as an error / require explicit family in provider_options + } +} + +pub fn messages_to_bedrock_body( + messages: &[Message], + tool_results_opt: Option<&[(ToolCall, ToolResult)]>, // For continue_ + config: &Config, +) -> Result { + let model_id = &config.model; + let model_family = get_bedrock_model_family(model_id); + + // Provider options from config + let provider_options: HashMap = config.provider_options.iter().map(|kv| (kv.key.clone(), kv.value.clone())).collect(); + + match model_family { + "claude" => { + // Based on Anthropic Claude on Bedrock format + // https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages.html + let mut anthropic_messages = Vec::new(); + let mut system_prompts = Vec::new(); + + for msg in messages { + if msg.role == Role::System { + for part in &msg.content { + if let ContentPart::Text(text) = part { + system_prompts.push(text.clone()); + } else { + return Err(Error{ code: ErrorCode::InvalidRequest, message: "Claude system prompts on Bedrock only support text".to_string(), provider_error_json: None }); + } + } + continue; + } + + let role_str = match msg.role { + Role::User => "user", + Role::Assistant => "assistant", + // Tool role messages for Claude are handled by appending tool_results content + Role::Tool => "user", // This needs careful handling with tool_results + Role::System => unreachable!(), // Handled above + }; + + let mut content_parts = Vec::new(); + for part in &msg.content { + match part { + ContentPart::Text(text) => content_parts.push(json!({ "type": "text", "text": text })), + ContentPart::Image(image_url) => { + // Assuming image_url.url is base64 encoded data for Claude on Bedrock + // e.g. "data:image/jpeg;base64,..." + // Claude on Bedrock expects: { "type": "image", "source": { "type": "base64", "media_type": "image/jpeg", "data": "..."}} + let (media_type, b64_data) = parse_data_url(&image_url.url)?; + content_parts.push(json!({ + "type": "image", + "source": { + "type": "base64", + "media_type": media_type, + "data": b64_data + } + })); + } + } + } + anthropic_messages.push(json!({ "role": role_str, "content": content_parts })); + } + + // Handle tool_results for continue_ case for Claude + if let Some(tool_results) = tool_results_opt { + for (tool_call, tool_result) in tool_results { + // Assistant's turn that requested the tool + anthropic_messages.push(json!({ + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": tool_call.id, + "name": tool_call.name, + "input": serde_json::from_str::(&tool_call.arguments_json).unwrap_or(json!({})) + } + ] + })); + // User's turn providing the tool result + let (result_content, is_error) = match tool_result { + ToolResult::Success(s) => (json!([{"type": "text", "text": s.result_json}]), false), + ToolResult::Error(e) => (json!([{"type": "text", "text": e.error_message}]), true), + }; + anthropic_messages.push(json!({ + "role": "user", // Or "tool" if Claude API differentiates. Docs say user. + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_call.id, + "content": result_content, // Claude on Bedrock v2.1 expects content to be a string for tool_result, or list of blocks + // For simplicity, passing JSON string as text. + // If it should be structured: + // "content": [{"type": "text", "text": if is_error { tool_result_error.error_message } else { tool_result_success.result_json }}], + "is_error": if is_error {Some(true)} else {None} + } + ] + })); + } + } + + + let mut body = json!({ + "anthropic_version": provider_options.get("anthropic_version").map_or("bedrock-2023-05-31", |s| s.as_str()), + "messages": anthropic_messages, + "max_tokens": config.max_tokens.unwrap_or(1024), // Claude calls it max_tokens + }); + + let body_obj = body.as_object_mut().unwrap(); + if !system_prompts.is_empty() { + body_obj.insert("system".to_string(), json!(system_prompts.join("\n"))); + } + if let Some(temp) = config.temperature { + body_obj.insert("temperature".to_string(), json!(temp)); + } + if let Some(stop_sequences) = &config.stop_sequences { + body_obj.insert("stop_sequences".to_string(), json!(stop_sequences)); + } + if !config.tools.is_empty() { + let bedrock_tools = tool_definitions_to_bedrock_tools(&config.tools, model_family)?; + body_obj.insert("tools".to_string(), json!(bedrock_tools)); + if let Some(tool_choice_str) = &config.tool_choice { + body_obj.insert("tool_choice".to_string(), convert_tool_choice_to_bedrock(tool_choice_str, model_family)); + } + } + // top_p, top_k for Claude + if let Some(top_p_str) = provider_options.get("top_p") { + if let Ok(top_p) = top_p_str.parse::() { body_obj.insert("top_p".to_string(), json!(top_p));} + } + if let Some(top_k_str) = provider_options.get("top_k") { + if let Ok(top_k) = top_k_str.parse::() { body_obj.insert("top_k".to_string(), json!(top_k));} + } + + Ok(json!(body_obj)) + } + "llama" => { + // Llama on Bedrock: expects a single string prompt. + // We need to construct this prompt from messages. + // Example: "[INST] User: ... [/INST] Assistant: ... [INST] User: ... [/INST]" + let mut prompt_string = String::new(); + if messages.len() == 1 && messages[0].role == Role::User && messages[0].content.len() == 1 { + if let ContentPart::Text(text) = &messages[0].content[0] { + prompt_string = text.clone(); // Simple case for playground-like single text prompt + } else { + return Err(Error { code: ErrorCode::InvalidRequest, message: "Llama on Bedrock with image requires specific prompt formatting not yet implemented simply.".to_string(), provider_error_json: None }); + } + } else { + // More complex conversation history for Llama chat models + for msg in messages { + let role_prefix = match msg.role { + Role::User => "User: ", + Role::Assistant => "Assistant: ", + Role::System => "", // System prompts are often prepended or handled differently + Role::Tool => "Tool output: ", // Llama might not directly support tool roles this way + }; + prompt_string.push_str(role_prefix); + for part in &msg.content { + if let ContentPart::Text(text) = part { + prompt_string.push_str(text); + prompt_string.push('\n'); + } else { + return Err(Error { code: ErrorCode::InvalidRequest, message: "Llama on Bedrock currently only supports text content in this simple conversion.".to_string(), provider_error_json: None }); + } + } + } + // Llama 3 specific format. Adjust if using Llama 2. + // This is a simplified formatter; a robust one is more complex. + let mut formatted_prompt = "<|begin_of_text|>".to_string(); + for msg in messages { + let role_str = match msg.role { + Role::User => "user", + Role::Assistant => "assistant", + Role::System => "system", + Role::Tool => "user", // Represent tool output as a user message or integrate differently + }; + formatted_prompt.push_str(&format!("<|start_header_id|>{}<|end_header_id|>\n\n", role_str)); + for part in &msg.content { + if let ContentPart::Text(text) = part { + formatted_prompt.push_str(text); + } + // Image handling for Llama needs specific model support & format + } + formatted_prompt.push_str("<|eot_id|>"); + } + if messages.last().map_or(false, |m| m.role != Role::Assistant) { + formatted_prompt.push_str("<|start_header_id|>assistant<|end_header_id|>\n\n"); + } + prompt_string = formatted_prompt; + } + + + let mut body = json!({ + "prompt": prompt_string, + "max_gen_len": config.max_tokens.unwrap_or(512), + }); + let body_obj = body.as_object_mut().unwrap(); + if let Some(temp) = config.temperature { + body_obj.insert("temperature".to_string(), json!(temp)); + } + if let Some(top_p_str) = provider_options.get("top_p") { // Llama uses top_p + if let Ok(top_p) = top_p_str.parse::() { body_obj.insert("top_p".to_string(), json!(top_p));} + } + // Llama does not support tools in the same way as Claude/OpenAI via Bedrock's direct invoke_model. + // Tool use with Llama on Bedrock typically involves function calling implemented on the client-side or via agents. + if !config.tools.is_empty() { + warn!("Tools are configured but Llama on Bedrock (via simple invoke_model) may not support them directly in the API call body. Tool processing might need to be client-side."); + } + + Ok(body) + } + // ... other model families ... + _ => Err(Error { + code: ErrorCode::InvalidRequest, + message: format!("Unsupported or unknown Bedrock model family for request conversion: {}", model_family), + provider_error_json: None, + }), + } +} + +fn parse_data_url(url: &str) -> Result<(String, String), Error> { + if !url.starts_with("data:") { + return Err(Error { code: ErrorCode::InvalidRequest, message: "Image URL is not a data URL".to_string(), provider_error_json: None }); + } + let parts: Vec<&str> = url.splitn(2, ',').collect(); + if parts.len() != 2 { + return Err(Error { code: ErrorCode::InvalidRequest, message: "Invalid data URL format".to_string(), provider_error_json: None }); + } + let header = parts[0]; // e.g., "data:image/jpeg;base64" + let b64_data = parts[1].to_string(); + + let mime_parts: Vec<&str> = header.trim_start_matches("data:").split(';').collect(); + let media_type = mime_parts.first().ok_or_else(|| Error { code: ErrorCode::InvalidRequest, message: "Missing media type in data URL".to_string(), provider_error_json: None })?.to_string(); + + if !mime_parts.contains(&"base64") { + return Err(Error { code: ErrorCode::InvalidRequest, message: "Data URL is not base64 encoded".to_string(), provider_error_json: None }); + } + + Ok((media_type, b64_data)) +} + + +// --- Response Conversion --- +pub fn bedrock_response_to_chat_event( + response_json: serde_json::Value, + model_id: &str, + base_metadata: ResponseMetadata, // provider_id, timestamp can be filled from SDK if available +) -> ChatEvent { + trace!("Converting Bedrock response to ChatEvent. Model ID: {}, Response: {}", model_id, response_json); + let model_family = get_bedrock_model_family(model_id); + + match model_family { + "claude" => { + // Example for Anthropic Claude on Bedrock + // https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages.html#model-parameters-anthropic-claude-messages-response + let id = response_json.get("id").and_then(|v| v.as_str()).unwrap_or_default().to_string(); + // let role = response_json.get("role").and_then(|v| v.as_str()).unwrap_or("assistant"); // Should be assistant + let stop_reason_str = response_json.get("stop_reason").and_then(|v| v.as_str()); + let stop_sequence = response_json.get("stop_sequence").and_then(|v| v.as_str()); // Can also be a stop reason + + let mut content_parts = Vec::new(); + let mut tool_call_parts: Vec = Vec::new(); + + if let Some(content_array) = response_json.get("content").and_then(|v| v.as_array()) { + for item_val in content_array { + let item_type = item_val.get("type").and_then(|v| v.as_str()); + match item_type { + Some("text") => { + if let Some(text) = item_val.get("text").and_then(|v| v.as_str()) { + content_parts.push(ContentPart::Text(text.to_string())); + } + } + Some("tool_use") => { + let tool_id = item_val.get("id").and_then(|v|v.as_str()).unwrap_or_default().to_string(); + let tool_name = item_val.get("name").and_then(|v|v.as_str()).unwrap_or_default().to_string(); + let tool_input = item_val.get("input").cloned().unwrap_or(json!({})); + tool_call_parts.push(ToolCall { + id: tool_id, + name: tool_name, + arguments_json: tool_input.to_string(), + }); + } + _ => warn!("Unknown content type in Claude Bedrock response: {:?}", item_val), + } + } + } + + let usage_obj = response_json.get("usage"); + let usage = usage_obj.map(|u| Usage { + input_tokens: u.get("input_tokens").and_then(|v| v.as_u64()).map(|v| v as u32), + output_tokens: u.get("output_tokens").and_then(|v| v.as_u64()).map(|v| v as u32), + total_tokens: None, // Bedrock Claude response doesn't provide total_tokens directly + }); + + let finish_reason = match stop_reason_str { + Some("end_turn") => Some(FinishReason::Stop), // Or Other if more appropriate + Some("tool_use") => Some(FinishReason::ToolCalls), + Some("max_tokens") => Some(FinishReason::Length), + Some("stop_sequence") => Some(FinishReason::Stop), + _ => stop_sequence.map(|_| FinishReason::Stop), // If stopped by stop_sequence + }; + + let metadata = ResponseMetadata { + finish_reason, + usage, + provider_id: Some(id.clone()), // Use response id as provider_id + ..base_metadata + }; + + if !tool_call_parts.is_empty() { + // If there are tool_use blocks, it's a tool request. + // Claude might also return text alongside tool_use, the interface needs to decide. + // Assuming if tool_calls are present, it's primarily a ToolRequest. + // Any text can be ignored or handled based on spec for golem:llm interface. + // For now, prioritizing tool_calls if present. + ChatEvent::ToolRequest(tool_call_parts) + } else if !content_parts.is_empty() { + ChatEvent::Message(CompleteResponse { + id, + content: content_parts, + tool_calls: Vec::new(), // No separate tool calls if primary content is text + metadata, + }) + } else { + warn!("Bedrock Claude response has no text content or tool_calls: {}", response_json); + ChatEvent::Error(Error{ code: ErrorCode::InternalError, message: "Empty response content from Bedrock Claude".to_string(), provider_error_json: Some(response_json.to_string())}) + } + } + "llama" => { + // Llama on Bedrock response: {"generation": "...", "prompt_token_count": ..., "generation_token_count": ..., "stop_reason": ...} + let generation = response_json.get("generation").and_then(|v| v.as_str()).unwrap_or_default().to_string(); + let input_tokens = response_json.get("prompt_token_count").and_then(|v| v.as_u64()).map(|v| v as u32); + let output_tokens = response_json.get("generation_token_count").and_then(|v| v.as_u64()).map(|v| v as u32); + let stop_reason_str = response_json.get("stop_reason").and_then(|v| v.as_str()); + + let usage = Some(Usage { + input_tokens, + output_tokens, + total_tokens: None, + }); + + let finish_reason = match stop_reason_str { + Some("stop") => Some(FinishReason::Stop), + Some("length") => Some(FinishReason::Length), + // Other Llama stop reasons: "content_filter" + _ => None, + }; + + let metadata = ResponseMetadata { + finish_reason, + usage, + ..base_metadata + }; + ChatEvent::Message(CompleteResponse { + id: String::new(), // Llama response doesn't have an 'id' field + content: vec![ContentPart::Text(generation)], + tool_calls: Vec::new(), // Llama invoke_model doesn't directly support tool calls in response + metadata: ResponseMetadata { + finish_reason, + usage, + provider_id: None, + timestamp: None, + provider_metadata_json: None, + }, + }) + } + _ => ChatEvent::Error(Error { + code: ErrorCode::InternalError, + message: format!("Response conversion not implemented for Bedrock model family: {}", model_family), + provider_error_json: Some(response_json.to_string()), + }), + } +} + + +// --- Streaming Conversion --- +pub fn bedrock_stream_chunk_to_stream_event( + chunk_json_str: &str, // This is the raw string data from the mimicked SSE + model_id: &str, +) -> Result, String> { + trace!("Converting Bedrock stream chunk. Model ID: {}, Chunk: {}", model_id, chunk_json_str); + let model_family = get_bedrock_model_family(model_id); + let chunk_json: serde_json::Value = serde_json::from_str(chunk_json_str) + .map_err(|e| format!("Failed to parse stream chunk JSON: {}. Chunk: {}", e, chunk_json_str))?; + + + match model_family { + "claude" => { + // Example for Anthropic Claude on Bedrock streaming + // https://docs.aws.amazon.com/bedrock/latest/userguide/streaming-invoke-model.html (general) + // https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages.html#model-parameters-anthropic-claude-messages-streaming + let event_type = chunk_json.get("type").and_then(|t| t.as_str()); + match event_type { + Some("message_start") => { + // Contains overall message metadata like role, id. + // Can be used to initialize response metadata if needed. + // let message = chunk_json.get("message").unwrap(); + // let id = message.get("id").and_then(|v|v.as_str()).unwrap_or_default(); + // store this id for the final StreamEvent::Finish + Ok(None) // No immediate delta from message_start itself typically + } + Some("content_block_start") => { + if let Some(content_block) = chunk_json.get("content_block") { + if content_block.get("type").and_then(|t| t.as_str()) == Some("tool_use") { + let tool_id = content_block.get("id").and_then(|v|v.as_str()).unwrap_or_default().to_string(); + let tool_name = content_block.get("name").and_then(|v|v.as_str()).unwrap_or_default().to_string(); + // Placeholder for partial tool call, arguments will come in input_json_delta + return Ok(Some(StreamEvent::Delta(StreamDelta { + content: None, + tool_calls: Some(vec![ToolCall { id: tool_id, name: tool_name, arguments_json: "".to_string() }]) // Mark start of tool call + }))); + } + } + Ok(None) + } + Some("content_block_delta") => { + if let Some(delta) = chunk_json.get("delta") { + let delta_type = delta.get("type").and_then(|t| t.as_str()); + match delta_type { + Some("text_delta") => { + let text = delta.get("text").and_then(|t| t.as_str()).unwrap_or_default(); + Ok(Some(StreamEvent::Delta(StreamDelta { + content: Some(vec![ContentPart::Text(text.to_string())]), + tool_calls: None, + }))) + } + Some("input_json_delta") => { + // This contains partial JSON for a tool_use block's input. + // The LlmChatStream state needs to aggregate these. + // The current StreamDelta only sends complete tool_calls. + // We might need a new StreamEvent variant for partial tool_arg_delta, + // or aggregate here and send once content_block_stop for this tool_use is received. + // For now, let's assume aggregation happens in the LlmChatStreamState implementation for Bedrock. + // This function should ideally just convert the direct delta. + // To fit StreamDelta, we'd need to know which tool_call this belongs to. + // The chunk includes an "index" field for the content_block. + let content_block_index = chunk_json.get("index").and_then(|i|i.as_u64()).unwrap_or(0); + let partial_json = delta.get("partial_json").and_then(|t| t.as_str()).unwrap_or_default(); + // This is a bit of a hack to fit into StreamDelta, signaling a partial argument update. + // The actual aggregation logic should be in BedrockChatStreamDirect. + Ok(Some(StreamEvent::Delta(StreamDelta { + content: None, + tool_calls: Some(vec![ToolCall { + id: format!("tool_idx_{}", content_block_index), // Placeholder ID based on index + name: "".to_string(), // Name would have come from content_block_start + arguments_json: partial_json.to_string(), // This is the partial delta + }]), + }))) + } + _ => Ok(None) + } + } else { Ok(None) } + } + Some("content_block_stop") => { + // Signals a content block (e.g., a tool_use block) is complete. + // If aggregating input_json_delta, this is where the full ToolCall could be emitted. + // For now, no direct event from this if deltas are handled separately. + Ok(None) + } + Some("message_delta") => { + // Contains "usage" and "stop_reason", "stop_sequence" + // These contribute to the final ResponseMetadata. + // Not a content delta itself. + // Can extract usage and stop_reason here to build final metadata. + // Example: + // if let Some(usage_val) = chunk_json.get("usage") { ... } + // if let Some(delta_val) = chunk_json.get("delta") { + // if let Some(stop_reason_str) = delta_val.get("stop_reason").and_then(|s|s.as_str()) { ... } + // } + Ok(None) + } + Some("message_stop") => { + // Final event in the stream for Claude. Contains final usage. + let usage_val = chunk_json.get("amazon-bedrock-invocationMetrics"); // For Claude 3 on Bedrock + let (input_tokens, output_tokens, latency_ms, first_byte_latency_ms) = if let Some(metrics) = usage_val { + (metrics.get("inputTokenCount").and_then(|v|v.as_u64()), + metrics.get("outputTokenCount").and_then(|v|v.as_u64()), + metrics.get("invocationLatency").and_then(|v|v.as_u64()), + metrics.get("firstByteLatency").and_then(|v|v.as_u64())) + } else { (None, None, None, None) }; + + + // The actual stop_reason should have been collected from message_delta. + // This event mainly confirms the end and provides final metrics. + let final_metadata = ResponseMetadata { + finish_reason: None, // Should be populated from earlier message_delta + usage: Some(Usage { + input_tokens: input_tokens.map(|v| v as u32), + output_tokens: output_tokens.map(|v| v as u32), + total_tokens: None, + }), + provider_id: None, // Should be populated from message_start + timestamp: None, + provider_metadata_json: Some(json!({ + "invocationLatency": latency_ms, + "firstByteLatency": first_byte_latency_ms + }).to_string()), + }; + Ok(Some(StreamEvent::Finish(final_metadata))) + } + Some("internal_failure") | Some("error") => { + let error_message = chunk_json.get("message").and_then(|m| m.as_str()).unwrap_or("Unknown streaming error").to_string(); + Err(error_message) + } + _ => { + warn!("Unknown Bedrock Claude stream event type: {:?}, Chunk: {}", event_type, chunk_json_str); + Ok(None) + } + } + } + "llama" => { + // Llama on Bedrock streaming response: {"generation": "...", "prompt_token_count": ..., "generation_token_count": ..., "stop_reason": ..., "amazon-bedrock-invocationMetrics": {...}} + // Each chunk is a JSON object. The "generation" field contains the text delta. + // The last chunk will have the stop_reason and metrics. + if let Some(generation_delta) = chunk_json.get("generation").and_then(|g| g.as_str()) { + // This is a content delta + Ok(Some(StreamEvent::Delta(StreamDelta { + content: Some(vec![ContentPart::Text(generation_delta.to_string())]), + tool_calls: None, + }))) + } else if chunk_json.get("stop_reason").is_some() { + // This is likely the final chunk with metadata + let input_tokens = chunk_json.get("prompt_token_count").and_then(|v| v.as_u64()).map(|v| v as u32); + let output_tokens = chunk_json.get("generation_token_count").and_then(|v| v.as_u64()).map(|v| v as u32); + let stop_reason_str = chunk_json.get("stop_reason").and_then(|v| v.as_str()); + let metrics = chunk_json.get("amazon-bedrock-invocationMetrics"); + + let finish_reason = match stop_reason_str { + Some("stop") => Some(FinishReason::Stop), + Some("length") => Some(FinishReason::Length), + _ => None, + }; + let usage = Some(Usage { input_tokens, output_tokens, total_tokens: None }); + + let metadata_json = metrics.map(|m| m.to_string()); + + Ok(Some(StreamEvent::Finish(ResponseMetadata { + finish_reason, + usage, + provider_id: None, // Llama stream doesn't provide an ID per message + timestamp: None, + provider_metadata_json: metadata_json, + }))) + } else if chunk_json.get("InternalServerException").is_some() || chunk_json.get("ModelStreamErrorException").is_some() { + Err(chunk_json.to_string()) + } + else { + warn!("Unknown Bedrock Llama stream chunk: {}", chunk_json_str); + Ok(None) + } + } + _ => Err(format!("Streaming conversion not implemented for Bedrock model family: {}", model_family)), + } +} + +fn tool_definitions_to_bedrock_tools(tools: &[ToolDefinition], model_family: &str) -> Result { + match model_family { + "claude" => { + let mut claude_tools = Vec::new(); + for tool_def in tools { + let params_schema: serde_json::Value = serde_json::from_str(&tool_def.parameters_schema) + .map_err(|e| Error { + code: ErrorCode::InvalidRequest, + message: format!("Invalid JSON schema for tool {}: {}", tool_def.name, e), + provider_error_json: Some(tool_def.parameters_schema.clone()), + })?; + claude_tools.push(json!({ + "name": tool_def.name, + "description": tool_def.description, + "input_schema": params_schema + })); + } + Ok(json!(claude_tools)) + } + // Other model families might have different tool formats or no direct support + _ => { + if !tools.is_empty() { + warn!("Tool definitions provided for model family '{}' which may not support them in this format.", model_family); + } + Ok(json!([])) // Empty array if not supported or unknown + } + } +} + +fn convert_tool_choice_to_bedrock(tool_choice_str: &str, model_family: &str) -> serde_json::Value { + match model_family { + "claude" => { + // Claude tool choice: {"type": "auto" | "any" | "tool", "name": "tool_name_if_type_is_tool"} + match tool_choice_str { + "auto" | "" => json!({"type": "auto"}), // Default to auto + "any" => json!({"type": "any"}), + "none" => json!({"type": "auto"}), + // If it's a specific tool name: + tool_name => json!({"type": "tool", "name": tool_name}), + } + } + _ => json!(null) // Or adapt for other families + } +} \ No newline at end of file diff --git a/llm-bedrock/src/lib.rs b/llm-bedrock/src/lib.rs new file mode 100644 index 0000000..2b664e7 --- /dev/null +++ b/llm-bedrock/src/lib.rs @@ -0,0 +1,261 @@ +mod client; +mod conversions; + +use std::cell::{Ref, RefCell, RefMut}; + +use golem_llm::chat_stream::{LlmChatStream, LlmChatStreamState}; +use golem_llm::durability::{DurableLLM, ExtendedGuest}; +use golem_llm::event_source::EventSource; +use golem_llm::golem::llm::llm::{ + ChatEvent, ChatStream, Config, Error as LlmError, ErrorCode, Guest, Message, ResponseMetadata, StreamEvent, ToolCall, ToolResult, +}; +use golem_llm::LOGGING_STATE; +use log::{debug, info, error}; + +use client::BedrockClient; +use conversions::{ + bedrock_response_to_chat_event, + messages_to_bedrock_body, +}; +use wit_bindgen_rt::async_support::block_on; + +const BEDROCK_DEFAULT_ACCEPT: &str = "application/json"; +const BEDROCK_DEFAULT_CONTENT_TYPE: &str = "application/json"; + + +// Custom stream implementation for Bedrock +pub struct BedrockChatStream { + stream: RefCell>, + failure: Option, + finished: RefCell, +} + +impl BedrockChatStream { + pub fn new(stream: EventSource) -> LlmChatStream { + LlmChatStream::new(BedrockChatStream { + stream: RefCell::new(Some(stream)), + failure: None, + finished: RefCell::new(false), + }) + } + + pub fn failed(error: LlmError) -> LlmChatStream { + LlmChatStream::new(BedrockChatStream { + stream: RefCell::new(None), + failure: Some(error), + finished: RefCell::new(false), + }) + } +} + +impl LlmChatStreamState for BedrockChatStream { + fn failure(&self) -> &Option { + &self.failure + } + + fn is_finished(&self) -> bool { + *self.finished.borrow() + } + + fn set_finished(&self) { + *self.finished.borrow_mut() = true; + } + + fn stream(&self) -> Ref> { + self.stream.borrow() + } + + fn stream_mut(&self) -> RefMut> { + self.stream.borrow_mut() + } + + fn decode_message(&self, _raw: &str) -> Result, String> { + // TODO: Implement Bedrock-specific message decoding + Ok(None) + } +} + +impl Drop for BedrockChatStream { + fn drop(&mut self) { + debug!("BedrockChatStream dropped"); + } +} + + +struct BedrockComponent; + +impl BedrockComponent { + // Helper to get region from provider_options or default AWS SDK behavior + fn get_aws_region(config: &Config) -> Option { + config.provider_options.iter() + .find(|kv| kv.key.eq_ignore_ascii_case("AWS_REGION") || kv.key.eq_ignore_ascii_case("REGION")) + .map(|kv| kv.value.clone()) + } +} + +impl Guest for BedrockComponent { + type ChatStream = LlmChatStream; + + fn send(messages: Vec, config: Config) -> ChatEvent { + LOGGING_STATE.with_borrow_mut(|state| state.init()); + info!("Bedrock: send called. Model: {}", config.model); + + block_on(async move { + let aws_region = Self::get_aws_region(&config); + match BedrockClient::new(aws_region).await { + Ok(client) => { + match messages_to_bedrock_body(&messages, None, &config) { + Ok(body) => { + match client + .invoke_model( + config.model.clone(), + body, + BEDROCK_DEFAULT_ACCEPT.to_string(), + BEDROCK_DEFAULT_CONTENT_TYPE.to_string(), + ) + .await + { + Ok(response_json) => bedrock_response_to_chat_event( + response_json, + &config.model, + ResponseMetadata { + finish_reason: None, + usage: None, + provider_id: None, + timestamp: None, + provider_metadata_json: None, + }, + ), + Err(e) => ChatEvent::Error(e), + } + } + Err(e) => ChatEvent::Error(e), + } + } + Err(e) => ChatEvent::Error(e), + } + }) + } + + fn continue_( + messages: Vec, + tool_results: Vec<(ToolCall, ToolResult)>, + config: Config, + ) -> ChatEvent { + LOGGING_STATE.with_borrow_mut(|state| state.init()); + info!("Bedrock: continue_ called. Model: {}", config.model); + block_on(async move { + let aws_region = Self::get_aws_region(&config); + match BedrockClient::new(aws_region).await { + Ok(client) => { + match messages_to_bedrock_body(&messages, Some(&tool_results), &config) { + Ok(body) => { + match client + .invoke_model( + config.model.clone(), + body, + BEDROCK_DEFAULT_ACCEPT.to_string(), + BEDROCK_DEFAULT_CONTENT_TYPE.to_string(), + ) + .await + { + Ok(response_json) => bedrock_response_to_chat_event( + response_json, + &config.model, + ResponseMetadata { + finish_reason: None, + usage: None, + provider_id: None, + timestamp: None, + provider_metadata_json: None, + }, + ), + Err(e) => ChatEvent::Error(e), + } + } + Err(e) => ChatEvent::Error(e), + } + } + Err(e) => ChatEvent::Error(e), + } + }) + } + + fn stream(messages: Vec, config: Config) -> ChatStream { + ChatStream::new(Self::unwrapped_stream(messages, config)) + } +} + +impl ExtendedGuest for BedrockComponent { + fn unwrapped_stream(messages: Vec, config: Config) -> Self::ChatStream { + LOGGING_STATE.with_borrow_mut(|state| state.init()); + info!("Bedrock: stream called. Model: {}", config.model); + + // The async block will attempt to set up the stream and resolve + // to Result. + // However, due to the mismatch between AWS SDK stream and EventSource requirements, + // it will currently return an Err. + let result_for_event_source: Result = block_on(async move { + let aws_region = Self::get_aws_region(&config); + let client = BedrockClient::new(aws_region).await?; // Returns LlmError on failure + + let body_json = messages_to_bedrock_body(&messages, None, &config)?; // Returns LlmError + + debug!("Requesting stream from Bedrock. Model: {}, Body: {}", config.model, body_json.to_string()); + + let sdk_stream_output = client + .invoke_model_with_response_stream( + config.model.clone(), + body_json, + BEDROCK_DEFAULT_ACCEPT.to_string(), // "application/json" + BEDROCK_DEFAULT_CONTENT_TYPE.to_string(), // "application/json" + ) + .await?; // This is LlmError from client.rs with mapped SDK errors + + // sdk_stream_output is InvokeModelWithResponseStreamOutput + // - sdk_stream_output.body is EventReceiver + // - sdk_stream_output.content_type is Option (expected to be "application/json") + + // PROBLEM POINT: + // golem_llm::event_source::EventSource::new() expects a `reqwest::Response` + // and internally checks for `Content-Type: text/event-stream`. + // The AWS SDK's `sdk_stream_output` does not directly provide a `reqwest::Response`. + // Also, Bedrock's `InvokeModelWithResponseStream` sends a stream of JSON objects, + // (matching the `Accept: application/json` header) not a standard SSE text/event-stream. + // This means EventSource's SSE parser would not be suitable for these JSON objects directly. + + // This step requires a significant adaptation layer or a change in how + // LlmChatStreamState consumes streams (i.e., not being tied to EventSource for SSE). + // For now, we return an error indicating this unimplemented/mismatched part. + Err(LlmError { + code: ErrorCode::InternalError, // Or perhaps ErrorCode::Unsupported if this adaptation is deemed out of scope + message: "Streaming from Bedrock SDK to golem-llm EventSource not yet fully implemented due to API/type mismatch.".to_string(), + provider_error_json: Some(format!( + "Bedrock stream content_type: {:?}. EventSource expects 'text/event-stream' and a reqwest::Response.", + sdk_stream_output.content_type + )), + }) + + // If adaptation was possible, it would look something like: + // let adapted_response_for_event_source = adapt_sdk_stream_to_reqwest_response(sdk_stream_output)?; + // EventSource::new(adapted_response_for_event_source) + // .map_err(|e| LlmError { /* convert event_source::error::Error to LlmError */ }) + }); + + match result_for_event_source { + Ok(event_source) => BedrockChatStream::new(event_source), + Err(llm_error) => { + error!("Failed to setup Bedrock stream: {}", llm_error.message); + BedrockChatStream::failed(llm_error) + } + } + } + + // Default retry_prompt from golem-llm/src/durability.rs is fine unless Bedrock needs a very specific format. +} + +// Wrap with DurableLLM +type DurableBedrockComponent = DurableLLM; + +// Export the durable component +golem_llm::export_llm!(DurableBedrockComponent with_types_in golem_llm); \ No newline at end of file diff --git a/llm-bedrock/wit/bedrock.wit b/llm-bedrock/wit/bedrock.wit new file mode 100644 index 0000000..95bea9d --- /dev/null +++ b/llm-bedrock/wit/bedrock.wit @@ -0,0 +1,5 @@ +package golem:llm-bedrock@1.0.0; + +world llm-library { + include golem:llm/llm-library@1.0.0; +} \ No newline at end of file diff --git a/test/components-rust/test-llm/golem.yaml b/test/components-rust/test-llm/golem.yaml index 1b12aeb..4f3bcb8 100644 --- a/test/components-rust/test-llm/golem.yaml +++ b/test/components-rust/test-llm/golem.yaml @@ -97,6 +97,28 @@ components: clean: - src/bindings.rs + bedrock-debug: + build: + - command: cargo component build --no-default-features --features bedrock # Assuming a 'bedrock' feature will be added to test_llm Cargo.toml + sources: + - src + - wit-generated + - ../../common-rust + targets: + - ../../target/wasm32-wasip1/debug/test_llm.wasm + - command: wac plug --plug ../../../target/wasm32-wasip1/debug/golem_llm_bedrock.wasm ../../target/wasm32-wasip1/debug/test_llm.wasm -o ../../target/wasm32-wasip1/debug/test_llm_plugged_bedrock.wasm + sources: + - ../../target/wasm32-wasip1/debug/test_llm.wasm + - ../../../target/wasm32-wasip1/debug/golem_llm_bedrock.wasm + targets: + - ../../target/wasm32-wasip1/debug/test_llm_plugged_bedrock.wasm + sourceWit: wit + generatedWit: wit-generated + componentWasm: ../../target/wasm32-wasip1/debug/test_llm_plugged_bedrock.wasm + linkedWasm: ../../golem-temp/components/test_llm_bedrock_debug.wasm # Ensure unique name + clean: + - src/bindings.rs + # RELEASE PROFILES openai-release: build: