diff --git a/.gitignore b/.gitignore index 96706905..d4e7eb35 100644 --- a/.gitignore +++ b/.gitignore @@ -40,7 +40,10 @@ bin *.env generate -test/integrate +test/integrate/person_list +test/integrate/project.py test/project_path.h +test/integrate/__pycache__/ +test/integrate/js/ .vscode/ -hash_hex/ +hash_hex/ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 84cece50..fec680a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) cmake_minimum_required(VERSION 3.12) -project(YPC VERSION 0.5.1) +project(YPC VERSION 0.5.2) list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) diff --git a/build.sh b/build.sh index f7bb2beb..36a10497 100755 --- a/build.sh +++ b/build.sh @@ -38,6 +38,7 @@ create_signed_so() { ["person_first_match"]="example/personlist/first_match/enclave/enclave.config.xml" ["person_first_match_multi"]="example/multi_personlist/first_match/enclave/enclave.config.xml" ["person_first_match_multi_offchain"]="example/multi_personlist/first_match/enclave_for_offchain/enclave.config.xml" + ["convert_sealed_file"]="example/convert/enclave/enclave.config.xml" ) for filename in ${!enclave_libs[@]}; do diff --git a/example/convert/CMakeLists.txt b/example/convert/CMakeLists.txt new file mode 100644 index 00000000..10a5584d --- /dev/null +++ b/example/convert/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(enclave) +add_subdirectory(enclave2) \ No newline at end of file diff --git a/example/convert/common.h b/example/convert/common.h new file mode 100644 index 00000000..1622d7d6 --- /dev/null +++ b/example/convert/common.h @@ -0,0 +1,7 @@ +#pragma once +#include "common_t.h" +#include "ypc/core/blockfile.h" +#include "ypc/core/byte.h" +#include "ypc/corecommon/package.h" +typedef ypc::blockfile<0x82, 1024 * 1024, 256 * 64 * 1024> file_t; + diff --git a/example/convert/common_t.h b/example/convert/common_t.h new file mode 100644 index 00000000..9cb25648 --- /dev/null +++ b/example/convert/common_t.h @@ -0,0 +1,36 @@ +#pragma once +#include +#include + +define_nt(RYXXBZ, std::string, "RYXXBZ"); +define_nt(XM, std::string, "XM"); +define_nt(CYM, std::string, "CYM"); +define_nt(XBDM, std::string, "XBDM"); +define_nt(FWXXBZ, std::string, "FWXXBZ"); +define_nt(XP, std::string, "XP"); +define_nt(DWMC, std::string, "DWMC"); +define_nt(ZJHM, std::string, "ZJHM"); +define_nt(GJDM, std::string, "GJDM"); +define_nt(MZDM, std::string, "MZDM"); +define_nt(JGSSXDM, std::string, "JGSSXDM"); +define_nt(HKXZFLYDM, std::string, "HKXZFLYDM"); +define_nt(HLXDM, std::string, "HLXDM"); +define_nt(HJDZ_XZQHDM, std::string, "HJDZXZQHDM"); +define_nt(SJJZD_XZQHDM, std::string, "SJJZDXZQHDM"); +define_nt(SJJZD_QHNXXDZ, std::string, "SJJZDQHNXXDZ"); +define_nt(XLDM, std::string, "XLDM"); +define_nt(TSSFDM, std::string, "TSSFDM"); +define_nt(CSQR, std::string, "CSQR"); +define_nt(LXDH, std::string, "LXDH"); +define_nt(HYZKDM, std::string, "HYZKDM"); +define_nt(DJR_XM, std::string, "DJR_XM"); +define_nt(DJR_GMSFZHM, std::string, "DJRGMSFZHM"); +define_nt(DJR_LXDH, std::string, "DJRLXDH"); +define_nt(GXSJ, std::string, "GXSJ"); +define_nt(SJZT, std::string, "SJZT"); + +typedef ff::util::ntobject< + RYXXBZ, XM, CYM, XBDM, FWXXBZ, XP, DWMC, ZJHM, GJDM, MZDM, JGSSXDM, + HKXZFLYDM, HLXDM, HJDZ_XZQHDM, SJJZD_XZQHDM, SJJZD_QHNXXDZ, XLDM, TSSFDM, + CSQR, LXDH, HYZKDM, DJR_XM, DJR_GMSFZHM, DJR_LXDH, GXSJ, SJZT> + row_t; diff --git a/example/convert/enclave/CMakeLists.txt b/example/convert/enclave/CMakeLists.txt new file mode 100644 index 00000000..5e657449 --- /dev/null +++ b/example/convert/enclave/CMakeLists.txt @@ -0,0 +1,13 @@ +set(T_SRCS eparser.cpp) + +add_ypc_applet(convert_sealed_file + CRYPTO stdeth + SRCS ${T_SRCS}) + +if(SGX_MODE STREQUAL "Debug") + enclave_sign(convert_sealed_file KEY enclave_private.pem + CONFIG enclave.config.debug.xml) +else() + enclave_sign(convert_sealed_file KEY enclave_private.pem + CONFIG enclave.config.xml) +endif() diff --git a/example/convert/enclave/convert_parser.h b/example/convert/enclave/convert_parser.h new file mode 100644 index 00000000..d970ae49 --- /dev/null +++ b/example/convert/enclave/convert_parser.h @@ -0,0 +1,193 @@ +#include "ypc/corecommon/package.h" +#include "ypc/stbox/ebyte.h" +#include "ypc/stbox/stx_common.h" +#ifdef EXAMPLE_FM_NORMAL +#include +typedef ypc::bytes bytes; +#else +#include "ypc/core_t/analyzer/data_source.h" +#include "ypc/stbox/tsgx/log.h" +typedef stbox::bytes bytes; +#endif +#include "user_type.h" +#include "ypc/corecommon/data_source.h" +#include "ypc/corecommon/to_type.h" +#include +#include +#include +#include + + +#include "ypc/core_t/analyzer/eparser_t_interface.h" +#include "ypc/common/crypto_prefix.h" +#include "ypc/corecommon/crypto/gmssl.h" +#include "ypc/corecommon/crypto/stdeth.h" +#include "ypc/corecommon/oram_types.h" +#include +#include + + +using oram_ntt = ypc::oram::nt; +using ntt = ypc::nt; + + + + + +define_nt(input_buf, std::string); +typedef ff::net::ntpackage<0, input_buf> input_buf_t; + + +class crypto_base { +public: + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) = 0; + virtual uint32_t hash_256(const bytes &msg, bytes &hash) = 0; +}; +using crypto_ptr_t = std::shared_ptr; +template class crypto_tool : public crypto_base { +public: + using crypto_t = Crypto; + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) { + return crypto_t::encrypt_message_with_prefix(public_key, data, prefix, + cipher); + } + virtual uint32_t hash_256(const bytes &msg, bytes &hash) { + return crypto_t::hash_256(msg, hash); + } +}; + + +class convert_parser { +public: + convert_parser() {} + convert_parser(ypc::data_source *source) : m_source(source){}; + + inline bytes do_parse(const bytes ¶m) { + bytes result; + + LOG(INFO) << "do convert_parse"; + ypc::to_type converter(m_source); + + crypto_ptr_t crypto_ptr = std::make_shared>(); + bytes pub_key(param); + + + + LOG(INFO) << "create id_map"; + + size_t batch_size = 0; + size_t item_size = 0; + uint64_t batch_num = 0; // the number of batch + uint64_t full_num = 0; + uint64_t last_num = 0; + uint64_t item_num = 0; + + std::vector id_map_array; + uint32_t batch_id = 1; + + hpda::processor::internal::filter_impl match2( + &converter, [&](const user_item_t &v) { + if(item_size == 0) { + typename ypc::cast_obj_to_package::type pt = v; + auto item_data = ypc::make_bytes::for_package(pt); + item_size = item_data.size(); + } + + std::string item_index_field = v.get(); + + input_buf_t item_index_field_pkg; + item_index_field_pkg.set(item_index_field); + bytes item_index_field_bytes = ypc::make_bytes::for_package(item_index_field_pkg); + bytes item_index_field_hash; + crypto_ptr->hash_256(item_index_field_bytes, item_index_field_hash); + + std::shared_ptr k_v(new oram_ntt::id_map_pair()); + k_v->set(item_index_field_hash, batch_id); + id_map_array.push_back(*k_v); + + ++item_num; + + batch_size += item_size; + if (batch_size >= ypc::utc::max_item_size) { + + if(full_num == 0) { + full_num = item_num; + } + + item_num = 0; + batch_size = 0; + + ++batch_id; + ++batch_num; + } + + return false; + }); + match2.get_engine()->run(); + + if(item_num > 0) { + last_num = item_num; + ++batch_num; + } + + oram_ntt::id_map_t id_map_pkg; + id_map_pkg.set(id_map_array); + id_map_array.clear(); + bytes id_map_bytes = ypc::make_bytes::for_package(id_map_pkg); + + + LOG(INFO) << "write header"; + + ypc::oram::header osf_header{}; + osf_header.block_num = batch_num; + uint32_t real_bucket_num = ceil(static_cast(osf_header.block_num) / ypc::oram::BucketSizeZ); + osf_header.level_num_L = ceil(log2(real_bucket_num + 1)) - 1; + osf_header.bucket_num_N = (1 << (osf_header.level_num_L + 1)) - 1; + osf_header.id_map_filepos = sizeof(osf_header); + osf_header.oram_tree_filepos = osf_header.id_map_filepos + id_map_bytes.size(); + + osf_header.item_size = item_size; + if(full_num > 0) { + osf_header.item_num_each_batch = full_num; + } else { + osf_header.item_num_each_batch = last_num; + } + + auto ret = stbox::ocall_cast(write_convert_data_structure) + (0, (uint8_t *)&osf_header, sizeof(osf_header)); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + + LOG(INFO) << "write id map"; + + int32_t id_map_size = 8000000; + for(int i = 0; i <= id_map_bytes.size(); i += id_map_size) { + if(i + id_map_size <= id_map_bytes.size()) { + ret = stbox::ocall_cast(write_convert_data_structure) + (osf_header.id_map_filepos + i, id_map_bytes.data() + i, id_map_size); + } else { + ret = stbox::ocall_cast(write_convert_data_structure) + (osf_header.id_map_filepos + i, id_map_bytes.data() + i, id_map_bytes.size() - i); + } + + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + } + + + return result; + } + +protected: + ypc::data_source *m_source; +}; diff --git a/example/convert/enclave/enclave.config.debug.xml b/example/convert/enclave/enclave.config.debug.xml new file mode 100644 index 00000000..b3c13bc5 --- /dev/null +++ b/example/convert/enclave/enclave.config.debug.xml @@ -0,0 +1,12 @@ + + 0 + 0 + 0x50000 + 0x80000000 + 10 + 1 + + 0 + 0 + 0xFFFFFFFF + diff --git a/example/convert/enclave/enclave.config.xml b/example/convert/enclave/enclave.config.xml new file mode 100644 index 00000000..08e3ae1b --- /dev/null +++ b/example/convert/enclave/enclave.config.xml @@ -0,0 +1,12 @@ + + 0 + 0 + 0x50000 + 0x80000000 + 10 + 1 + + 1 + 0 + 0xFFFFFFFF + diff --git a/example/convert/enclave/enclave.lds b/example/convert/enclave/enclave.lds new file mode 100644 index 00000000..0d5614f5 --- /dev/null +++ b/example/convert/enclave/enclave.lds @@ -0,0 +1,11 @@ +enclave.so +{ + global: + g_global_data_sim; + g_global_data; + enclave_entry; + g_peak_heap_used; + g_peak_rsrv_mem_committed; + local: + *; +}; diff --git a/example/convert/enclave/enclave_debug.lds b/example/convert/enclave/enclave_debug.lds new file mode 100644 index 00000000..0d5614f5 --- /dev/null +++ b/example/convert/enclave/enclave_debug.lds @@ -0,0 +1,11 @@ +enclave.so +{ + global: + g_global_data_sim; + g_global_data; + enclave_entry; + g_peak_heap_used; + g_peak_rsrv_mem_committed; + local: + *; +}; diff --git a/example/convert/enclave/enclave_private.pem b/example/convert/enclave/enclave_private.pem new file mode 100644 index 00000000..529d07be --- /dev/null +++ b/example/convert/enclave/enclave_private.pem @@ -0,0 +1,39 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIG4gIBAAKCAYEAroOogvsj/fZDZY8XFdkl6dJmky0lRvnWMmpeH41Bla6U1qLZ +AmZuyIF+mQC/cgojIsrBMzBxb1kKqzATF4+XwPwgKz7fmiddmHyYz2WDJfAjIveJ +ZjdMjM4+EytGlkkJ52T8V8ds0/L2qKexJ+NBLxkeQLfV8n1mIk7zX7jguwbCG1Pr +nEMdJ3Sew20vnje+RsngAzdPChoJpVsWi/K7cettX/tbnre1DL02GXc5qJoQYk7b +3zkmhz31TgFrd9VVtmUGyFXAysuSAb3EN+5VnHGr0xKkeg8utErea2FNtNIgua8H +ONfm9Eiyaav1SVKzPHlyqLtcdxH3I8Wg7yqMsaprZ1n5A1v/levxnL8+It02KseD +5HqV4rf/cImSlCt3lpRg8U5E1pyFQ2IVEC/XTDMiI3c+AR+w2jSRB3Bwn9zJtFlW +KHG3m1xGI4ck+Lci1JvWWLXQagQSPtZTsubxTQNx1gsgZhgv1JHVZMdbVlAbbRMC +1nSuJNl7KPAS/VfzAgEDAoIBgHRXxaynbVP5gkO0ug6Qw/E27wzIw4SmjsxG6Wpe +K7kfDeRskKxESdsA/xCrKkwGwhcx1iIgS5+Qscd1Yg+1D9X9asd/P7waPmWoZd+Z +AhlKwhdPsO7PiF3e1AzHhGQwsUTt/Y/aSI1MpHBvy2/s1h9mFCslOUxTmWw0oj/Q +ldIEgWeNR72CE2+jFIJIyml6ftnb6qzPiga8Bm48ubKh0kvySOqnkmnPzgh+JBD6 +JnBmtZbfPT97bwTT+N6rnPqOOApvfHPf15kWI8yDbprG1l4OCUaIUH1AszxLd826 +5IPM+8gINLRDP1MA6azECPjTyHXhtnSIBZCyWSVkc05vYmNXYUNiXWMajcxW9M02 +wKzFELO8NCEAkaTPxwo4SCyIjUxiK1LbQ9h8PSy4c1+gGP4LAMR8xqP4QKg6zdu9 +osUGG/xRe/uufgTBFkcjqBHtK5L5VI0jeNIUAgW/6iNbYXjBMJ0GfauLs+g1VsOm +WfdgXzsb9DYdMa0OXXHypmV4GwKBwQDUwQj8RKJ6c8cT4vcWCoJvJF00+RFL+P3i +Gx2DLERxRrDa8AVGfqaCjsR+3vLgG8V/py+z+dxZYSqeB80Qeo6PDITcRKoeAYh9 +xlT3LJOS+k1cJcEmlbbO2IjLkTmzSwa80fWexKu8/Xv6vv15gpqYl1ngYoqJM3pd +vzmTIOi7MKSZ0WmEQavrZj8zK4endE3v0eAEeQ55j1GImbypSf7Idh7wOXtjZ7WD +Dg6yWDrri+AP/L3gClMj8wsAxMV4ZR8CgcEA0fzDHkFa6raVOxWnObmRoDhAtE0a +cjUj976NM5yyfdf2MrKy4/RhdTiPZ6b08/lBC/+xRfV3xKVGzacm6QjqjZrUpgHC +0LKiZaMtccCJjLtPwQd0jGQEnKfMFaPsnhOc5y8qVkCzVOSthY5qhz0XNotHHFmJ +gffVgB0iqrMTvSL7IA2yqqpOqNRlhaYhNl8TiFP3gIeMtVa9rZy31JPgT2uJ+kfo +gV7sdTPEjPWZd7OshGxWpT6QfVDj/T9T7L6tAoHBAI3WBf2DFvxNL2KXT2QHAZ9t +k3imC4f7U+wSE6zILaDZyzygA4RUbwG0gv8/TJVn2P/Eynf76DuWHGlaiLWnCbSz +Az2DHBQBBaku409zDQym3j1ugMRjzzSQWzJg0SIyBH3hTmnYcn3+Uqcp/lEBvGW6 +O+rsXFt3pukqJmIV8HzLGGaLm62BHUeZf3dyWm+i3p/hQAL7Xvu04QW70xuGqdr5 +afV7p5eaeQIJXyGQJ0eylV/90+qxjMKiB1XYg6WYvwKBwQCL/ddpgOdHJGN8uRom +e7Zq0Csi3hGheMKlKbN3vcxT5U7MdyHtTZZOJbTvxKNNUNYH/8uD+PqDGNneb29G +BfGzvI3EASyLIcGZF3OhKwZd0jUrWk2y7Vhob91jwp2+t73vdMbkKyI4mHOuXvGv +fg95si9oO7EBT+Oqvhccd2J+F1IVXncccYnF4u5ZGWt5lLewN/pVr7MjjykeaHqN +t+rfnQam2psA6fL4zS2zTmZPzR2tnY8Y1GBTi0Ko1OKd1HMCgcAb5cB/7/AQlhP9 +yQa04PLH9ygQkKKptZp7dy5WcWRx0K/hAHRoi2aw1wZqfm7VBNu2SLcs90kCCCxp +6C5sfJi6b8NpNbIPC+sc9wsFr7pGo9SFzQ78UlcWYK2Gu2FxlMjonhka5hvo4zvg +WxlpXKEkaFt3gLd92m/dMqBrHfafH7VwOJY2zT3WIpjwuk0ZzmRg5p0pG/svVQEH +NZmwRwlopysbR69B/n1nefJ84UO50fLh5s5Zr3gBRwbWNZyzhXk= +-----END RSA PRIVATE KEY----- diff --git a/example/convert/enclave/eparser.cpp b/example/convert/enclave/eparser.cpp new file mode 100644 index 00000000..98e59883 --- /dev/null +++ b/example/convert/enclave/eparser.cpp @@ -0,0 +1,13 @@ +#include "ypc/core_t/analyzer/algo_wrapper.h" +#include "ypc/core_t/analyzer/macro.h" +#include "ypc/corecommon/crypto/stdeth.h" +#include "ypc/corecommon/crypto/gmssl.h" + +#include "convert_parser.h" + +ypc::algo_wrapper> + pw; + +YPC_PARSER_IMPL(pw); \ No newline at end of file diff --git a/example/convert/enclave/user_type.h b/example/convert/enclave/user_type.h new file mode 100644 index 00000000..8920839d --- /dev/null +++ b/example/convert/enclave/user_type.h @@ -0,0 +1,7 @@ +#pragma once +#include "../common_t.h" +#include +#include +#include + +typedef row_t user_item_t; diff --git a/example/convert/enclave2/CMakeLists.txt b/example/convert/enclave2/CMakeLists.txt new file mode 100644 index 00000000..da106139 --- /dev/null +++ b/example/convert/enclave2/CMakeLists.txt @@ -0,0 +1,13 @@ +set(T_SRCS eparser.cpp) + +add_ypc_applet(convert_sealed_file2 + CRYPTO stdeth + SRCS ${T_SRCS}) + +if(SGX_MODE STREQUAL "Debug") + enclave_sign(convert_sealed_file2 KEY enclave_private.pem + CONFIG enclave.config.debug.xml) +else() + enclave_sign(convert_sealed_file2 KEY enclave_private.pem + CONFIG enclave.config.xml) +endif() diff --git a/example/convert/enclave2/convert_parser2.h b/example/convert/enclave2/convert_parser2.h new file mode 100644 index 00000000..e08f4186 --- /dev/null +++ b/example/convert/enclave2/convert_parser2.h @@ -0,0 +1,479 @@ +#include "ypc/corecommon/package.h" +#include "ypc/stbox/ebyte.h" +#include "ypc/stbox/stx_common.h" +#ifdef EXAMPLE_FM_NORMAL +#include +typedef ypc::bytes bytes; +#else +#include "ypc/core_t/analyzer/data_source.h" +#include "ypc/stbox/tsgx/log.h" +typedef stbox::bytes bytes; +#endif +#include "user_type.h" +#include "ypc/corecommon/data_source.h" +#include "ypc/corecommon/to_type.h" +#include +#include +#include +#include + + +#include "ypc/core_t/analyzer/eparser_t_interface.h" +#include "ypc/common/crypto_prefix.h" +#include "ypc/corecommon/crypto/gmssl.h" +#include "ypc/corecommon/crypto/stdeth.h" +#include "ypc/corecommon/oram_types.h" +#include +#include + + +using oram_ntt = ypc::oram::nt; +using ntt = ypc::nt; + + + + + +define_nt(input_buf, std::string); +typedef ff::net::ntpackage<0, input_buf> input_buf_t; + + +class crypto_base { +public: + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) = 0; + virtual uint32_t hash_256(const bytes &msg, bytes &hash) = 0; +}; +using crypto_ptr_t = std::shared_ptr; +template class crypto_tool : public crypto_base { +public: + using crypto_t = Crypto; + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) { + return crypto_t::encrypt_message_with_prefix(public_key, data, prefix, + cipher); + } + virtual uint32_t hash_256(const bytes &msg, bytes &hash) { + return crypto_t::hash_256(msg, hash); + } +}; + + +bytes random_string(size_t len) { + std::string ret(len, '0'); + static std::default_random_engine generator; + static std::uniform_int_distribution distribution(int('a'), int('z')); + static auto rand = std::bind(distribution, generator); + + for (size_t i = 0; i < len; i++) { + ret[i] = rand(); + } + return bytes(ret.data(), ret.size()); +} + +bool push_dummy_block(std::vector& bucket_array, bytes &data_hash, + uint8_t count, uint64_t item_num_each_batch, uint64_t item_size, + const crypto_ptr_t &crypto_ptr, const bytes &public_key) { + for(uint8_t i = 0; i < count; ++i) { + oram_ntt::block_t b_block; + + std::vector dummy_batch; + for(uint32_t j = 0; j < item_num_each_batch; ++j) { + bytes dummy_item = random_string(item_size); + dummy_batch.push_back(dummy_item); + bytes k_hash = data_hash + dummy_item; + crypto_ptr->hash_256(k_hash, data_hash); + } + + bytes encrypted_dummy_batch; + bytes dummy_batch_str = + ypc::make_bytes::for_package(dummy_batch); + + // encrypt dummy batch + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, dummy_batch_str, ypc::utc::crypto_prefix_arbitrary, encrypted_dummy_batch); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + b_block.set(0, 0, 0, encrypted_dummy_batch); + bucket_array.push_back(b_block); + } + return true; +} + +uint32_t get_leaf_label(uint32_t bucket_index, uint8_t level_num_L) { + // leftmost leaf node + uint32_t leftmost_leaf_index = (1 << level_num_L) - 1; + if(bucket_index >= leftmost_leaf_index) { + return bucket_index - leftmost_leaf_index + 1; + } + + // randomly select a path to the leaf node + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(0, 1); + + if(dis(gen) == 0) { + return get_leaf_label(2 * bucket_index + 1, level_num_L); + } + return get_leaf_label(2 * bucket_index + 2, level_num_L); +} + +bool push_real_block(std::vector& bucket_array, bytes &data_hash, + uint32_t& block_id_value, uint32_t bucket_index, + std::vector &position_map_array, uint8_t level_num_L, + std::vector &batch, uint32_t &batch_str_size, + uint64_t item_num_each_batch, uint64_t item_size, + const crypto_ptr_t &crypto_ptr, const bytes &public_key) { + oram_ntt::block_t b_block; + uint32_t valid_item_num = batch.size(); + for(uint32_t i = 0; i < item_num_each_batch - valid_item_num; ++i) { + ypc::bytes item = random_string(item_size); + batch.push_back(item); + } + + for(auto &item : batch) { + ypc::bytes k_hash = data_hash + item; + crypto_ptr->hash_256(k_hash, data_hash); + } + + bytes encrypted_batch; + ypc::bytes batch_str = + ypc::make_bytes::for_package(batch); + // encrypt batch + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, batch_str, ypc::utc::crypto_prefix_arbitrary, encrypted_batch); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + if(batch_str_size != encrypted_batch.size()) { + batch_str_size = encrypted_batch.size(); + } + + uint32_t b_leaf_label = get_leaf_label(bucket_index, level_num_L); + position_map_array[block_id_value] = b_leaf_label; + b_block.set + (block_id_value++, b_leaf_label, valid_item_num, encrypted_batch); + bucket_array.push_back(b_block); + + return true; +} + + +class convert_parser2 { +public: + convert_parser2() {} + convert_parser2(ypc::data_source *source) : m_source(source){}; + + inline bytes do_parse(const bytes ¶m) { + bytes result; + LOG(INFO) << "do convert_parse2"; + ypc::to_type converter(m_source); + + crypto_ptr_t crypto_ptr = std::make_shared>(); + bytes pub_key(param); + + + // 1. read header + ypc::oram::header osf_header{}; + + auto ret = stbox::ocall_cast(download_convert_params_ocall) + (&osf_header.block_num, &osf_header.oram_tree_filepos, &osf_header.item_num_each_batch, &osf_header.item_size); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "download_convert_params_ocall fail!"; + return result; + } + + uint32_t real_bucket_num = ceil(static_cast(osf_header.block_num) / ypc::oram::BucketSizeZ); + osf_header.level_num_L = ceil(log2(real_bucket_num + 1)) - 1; + osf_header.bucket_num_N = (1 << (osf_header.level_num_L + 1)) - 1; + osf_header.id_map_filepos = sizeof(osf_header); + + + // 2. write ORAM tree + LOG(INFO) << "write ORAM tree"; + std::vector data_hash_array; + + // from which bucket to start writing real blocks + uint8_t lastbucket_realblocknum = osf_header.block_num % ypc::oram::BucketSizeZ; + uint32_t bucket_index = 0; // bucket index in ORAM tree + uint32_t block_id_value = 1; // block_id_value <= osf_header.block_num + + + // 2.1 write buckets full of dummy blocks + LOG(INFO) << "write buckets full of dummy blocks"; + // osf.seekp(osf_header.oram_tree_filepos, osf.beg); + int64_t filepos = osf_header.oram_tree_filepos; + for(uint32_t i = 0; i < osf_header.bucket_num_N - real_bucket_num; ++i) { + std::vector bucket_array; + bytes data_hash; + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + bool retf = push_dummy_block(bucket_array, data_hash, ypc::oram::BucketSizeZ, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + if(!retf) { + LOG(ERROR) << "push_dummy_block fail!"; + return result; + } + + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = ypc::make_bytes::for_package(bucket_pkg); + + // secondary encryption on the serialized bucket + // in order to encrypt the mapping relationship between block_id and leaf_label + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return result; + } + + ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + + filepos += encrypted_bucket_bytes.size(); + data_hash_array.push_back(data_hash); + ++bucket_index; + } + + + + std::vector batch; + std::vector position_map_array(osf_header.block_num + 1, 0); + + + LOG(INFO) << "write real blocks"; + // 2.2 write the bucket that contains both real and dummy blocks + std::vector bucket_array; + bytes data_hash; + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + int i = ypc::oram::BucketSizeZ; + + if(lastbucket_realblocknum != 0) { + --real_bucket_num; + push_dummy_block(bucket_array, data_hash, + ypc::oram::BucketSizeZ - lastbucket_realblocknum, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + i = lastbucket_realblocknum; + } + + bool break_flag = false; + int j = 0; + batch.clear(); + + + // 2.3 write buckets full of real blocks + hpda::processor::internal::filter_impl match3( + &converter, [&](const user_item_t &v) { + typename ypc::cast_obj_to_package::type pt = v; + auto item_data = ypc::make_bytes::for_package(pt); + + if(break_flag) { + return false; + } + + batch.push_back(item_data); + ++j; + + if(j == osf_header.item_num_each_batch) { + + bool retf = push_real_block(bucket_array, data_hash, block_id_value, bucket_index, + position_map_array, osf_header.level_num_L, batch, osf_header.batch_str_size, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + if(!retf) { + LOG(ERROR) << "push_dummy_block fail!"; + break_flag = true; + return false; + } + + --i; + + if(i == 0) { + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = ypc::make_bytes::for_package(bucket_pkg); + + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + break_flag = true; + return false; + } + + if(osf_header.bucket_str_size != encrypted_bucket_bytes.size()) { + osf_header.bucket_str_size = encrypted_bucket_bytes.size(); + } + + auto ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + break_flag = true; + return false; + } + + + filepos += encrypted_bucket_bytes.size(); + data_hash_array.push_back(data_hash); + ++bucket_index; + bucket_array.clear(); + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + + i = ypc::oram::BucketSizeZ; + } + + batch.clear(); + j = 0; + } + + return false; + }); + match3.get_engine()->run(); + + if(break_flag) { + LOG(ERROR) << "write real blocks fail!"; + return result; + } + + // 2.4 write the last data block + // The number of valid rows in the last data block may be less than item_num_each_batch + if(j > 0) { + bool retf = push_real_block(bucket_array, data_hash, block_id_value, bucket_index, + position_map_array, osf_header.level_num_L, batch, osf_header.batch_str_size, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + if(!retf) { + LOG(ERROR) << "push_dummy_block fail!"; + break_flag = true; + return result; + } + + --i; + + if(i == 0) { + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = ypc::make_bytes::for_package(bucket_pkg); + + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + break_flag = true; + return result; + } + + if(osf_header.bucket_str_size != encrypted_bucket_bytes.size()) { + osf_header.bucket_str_size = encrypted_bucket_bytes.size(); + } + + auto ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + break_flag = true; + return result; + } + + filepos += encrypted_bucket_bytes.size(); + data_hash_array.push_back(data_hash); + ++bucket_index; + bucket_array.clear(); + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + } + } + + + LOG(INFO) << "write real blocks done"; + + + // 3. write position_map + osf_header.position_map_filepos = filepos; + oram_ntt::position_map_t position_map_pkg; + position_map_pkg.set(position_map_array); + bytes position_map_bytes = ypc::make_bytes::for_package(position_map_pkg); + ypc::bytes encrypted_position_map_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, position_map_bytes, ypc::utc::crypto_prefix_arbitrary, encrypted_position_map_bytes); + if (status != 0u) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return result; + } + + ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_position_map_bytes.data(), encrypted_position_map_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + filepos += encrypted_position_map_bytes.size(); + + + // 4. write merkle tree + osf_header.merkle_tree_filepos = filepos; + + for(int i = (1 << osf_header.level_num_L) - 2; i >= 0; --i) { + ypc::bytes k_hash = data_hash_array[i] + data_hash_array[2*i + 1]; + crypto_ptr->hash_256(k_hash, data_hash_array[i]); + + k_hash = data_hash_array[i] + data_hash_array[2*i + 2]; + crypto_ptr->hash_256(k_hash, data_hash_array[i]); + } + + for(auto &data_hash : data_hash_array) { + ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, data_hash.data(), data_hash.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + filepos += data_hash.size(); + } + + + // 5. update and write osf_header + osf_header.stash_filepos = filepos; + osf_header.item_num_each_batch = 0; + osf_header.item_size = 0; + + ret = stbox::ocall_cast(write_convert_data_structure) + (0, (uint8_t *)&osf_header, sizeof(osf_header)); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + + + LOG(INFO) << "convert_parse2 done"; + + return result; + } + + +protected: + ypc::data_source *m_source; +}; diff --git a/example/convert/enclave2/enclave.config.debug.xml b/example/convert/enclave2/enclave.config.debug.xml new file mode 100644 index 00000000..471a1132 --- /dev/null +++ b/example/convert/enclave2/enclave.config.debug.xml @@ -0,0 +1,12 @@ + + 0 + 0 + 0x50000 + 0x5000000 + 10 + 1 + + 0 + 0 + 0xFFFFFFFF + diff --git a/example/convert/enclave2/enclave.config.xml b/example/convert/enclave2/enclave.config.xml new file mode 100644 index 00000000..df005eb0 --- /dev/null +++ b/example/convert/enclave2/enclave.config.xml @@ -0,0 +1,12 @@ + + 0 + 0 + 0x50000 + 0x5000000 + 10 + 1 + + 1 + 0 + 0xFFFFFFFF + diff --git a/example/convert/enclave2/enclave.lds b/example/convert/enclave2/enclave.lds new file mode 100644 index 00000000..0d5614f5 --- /dev/null +++ b/example/convert/enclave2/enclave.lds @@ -0,0 +1,11 @@ +enclave.so +{ + global: + g_global_data_sim; + g_global_data; + enclave_entry; + g_peak_heap_used; + g_peak_rsrv_mem_committed; + local: + *; +}; diff --git a/example/convert/enclave2/enclave_debug.lds b/example/convert/enclave2/enclave_debug.lds new file mode 100644 index 00000000..0d5614f5 --- /dev/null +++ b/example/convert/enclave2/enclave_debug.lds @@ -0,0 +1,11 @@ +enclave.so +{ + global: + g_global_data_sim; + g_global_data; + enclave_entry; + g_peak_heap_used; + g_peak_rsrv_mem_committed; + local: + *; +}; diff --git a/example/convert/enclave2/enclave_private.pem b/example/convert/enclave2/enclave_private.pem new file mode 100644 index 00000000..529d07be --- /dev/null +++ b/example/convert/enclave2/enclave_private.pem @@ -0,0 +1,39 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIG4gIBAAKCAYEAroOogvsj/fZDZY8XFdkl6dJmky0lRvnWMmpeH41Bla6U1qLZ +AmZuyIF+mQC/cgojIsrBMzBxb1kKqzATF4+XwPwgKz7fmiddmHyYz2WDJfAjIveJ +ZjdMjM4+EytGlkkJ52T8V8ds0/L2qKexJ+NBLxkeQLfV8n1mIk7zX7jguwbCG1Pr +nEMdJ3Sew20vnje+RsngAzdPChoJpVsWi/K7cettX/tbnre1DL02GXc5qJoQYk7b +3zkmhz31TgFrd9VVtmUGyFXAysuSAb3EN+5VnHGr0xKkeg8utErea2FNtNIgua8H +ONfm9Eiyaav1SVKzPHlyqLtcdxH3I8Wg7yqMsaprZ1n5A1v/levxnL8+It02KseD +5HqV4rf/cImSlCt3lpRg8U5E1pyFQ2IVEC/XTDMiI3c+AR+w2jSRB3Bwn9zJtFlW +KHG3m1xGI4ck+Lci1JvWWLXQagQSPtZTsubxTQNx1gsgZhgv1JHVZMdbVlAbbRMC +1nSuJNl7KPAS/VfzAgEDAoIBgHRXxaynbVP5gkO0ug6Qw/E27wzIw4SmjsxG6Wpe +K7kfDeRskKxESdsA/xCrKkwGwhcx1iIgS5+Qscd1Yg+1D9X9asd/P7waPmWoZd+Z +AhlKwhdPsO7PiF3e1AzHhGQwsUTt/Y/aSI1MpHBvy2/s1h9mFCslOUxTmWw0oj/Q +ldIEgWeNR72CE2+jFIJIyml6ftnb6qzPiga8Bm48ubKh0kvySOqnkmnPzgh+JBD6 +JnBmtZbfPT97bwTT+N6rnPqOOApvfHPf15kWI8yDbprG1l4OCUaIUH1AszxLd826 +5IPM+8gINLRDP1MA6azECPjTyHXhtnSIBZCyWSVkc05vYmNXYUNiXWMajcxW9M02 +wKzFELO8NCEAkaTPxwo4SCyIjUxiK1LbQ9h8PSy4c1+gGP4LAMR8xqP4QKg6zdu9 +osUGG/xRe/uufgTBFkcjqBHtK5L5VI0jeNIUAgW/6iNbYXjBMJ0GfauLs+g1VsOm +WfdgXzsb9DYdMa0OXXHypmV4GwKBwQDUwQj8RKJ6c8cT4vcWCoJvJF00+RFL+P3i +Gx2DLERxRrDa8AVGfqaCjsR+3vLgG8V/py+z+dxZYSqeB80Qeo6PDITcRKoeAYh9 +xlT3LJOS+k1cJcEmlbbO2IjLkTmzSwa80fWexKu8/Xv6vv15gpqYl1ngYoqJM3pd +vzmTIOi7MKSZ0WmEQavrZj8zK4endE3v0eAEeQ55j1GImbypSf7Idh7wOXtjZ7WD +Dg6yWDrri+AP/L3gClMj8wsAxMV4ZR8CgcEA0fzDHkFa6raVOxWnObmRoDhAtE0a +cjUj976NM5yyfdf2MrKy4/RhdTiPZ6b08/lBC/+xRfV3xKVGzacm6QjqjZrUpgHC +0LKiZaMtccCJjLtPwQd0jGQEnKfMFaPsnhOc5y8qVkCzVOSthY5qhz0XNotHHFmJ +gffVgB0iqrMTvSL7IA2yqqpOqNRlhaYhNl8TiFP3gIeMtVa9rZy31JPgT2uJ+kfo +gV7sdTPEjPWZd7OshGxWpT6QfVDj/T9T7L6tAoHBAI3WBf2DFvxNL2KXT2QHAZ9t +k3imC4f7U+wSE6zILaDZyzygA4RUbwG0gv8/TJVn2P/Eynf76DuWHGlaiLWnCbSz +Az2DHBQBBaku409zDQym3j1ugMRjzzSQWzJg0SIyBH3hTmnYcn3+Uqcp/lEBvGW6 +O+rsXFt3pukqJmIV8HzLGGaLm62BHUeZf3dyWm+i3p/hQAL7Xvu04QW70xuGqdr5 +afV7p5eaeQIJXyGQJ0eylV/90+qxjMKiB1XYg6WYvwKBwQCL/ddpgOdHJGN8uRom +e7Zq0Csi3hGheMKlKbN3vcxT5U7MdyHtTZZOJbTvxKNNUNYH/8uD+PqDGNneb29G +BfGzvI3EASyLIcGZF3OhKwZd0jUrWk2y7Vhob91jwp2+t73vdMbkKyI4mHOuXvGv +fg95si9oO7EBT+Oqvhccd2J+F1IVXncccYnF4u5ZGWt5lLewN/pVr7MjjykeaHqN +t+rfnQam2psA6fL4zS2zTmZPzR2tnY8Y1GBTi0Ko1OKd1HMCgcAb5cB/7/AQlhP9 +yQa04PLH9ygQkKKptZp7dy5WcWRx0K/hAHRoi2aw1wZqfm7VBNu2SLcs90kCCCxp +6C5sfJi6b8NpNbIPC+sc9wsFr7pGo9SFzQ78UlcWYK2Gu2FxlMjonhka5hvo4zvg +WxlpXKEkaFt3gLd92m/dMqBrHfafH7VwOJY2zT3WIpjwuk0ZzmRg5p0pG/svVQEH +NZmwRwlopysbR69B/n1nefJ84UO50fLh5s5Zr3gBRwbWNZyzhXk= +-----END RSA PRIVATE KEY----- diff --git a/example/convert/enclave2/eparser.cpp b/example/convert/enclave2/eparser.cpp new file mode 100644 index 00000000..1940f920 --- /dev/null +++ b/example/convert/enclave2/eparser.cpp @@ -0,0 +1,13 @@ +#include "ypc/core_t/analyzer/algo_wrapper.h" +#include "ypc/core_t/analyzer/macro.h" +#include "ypc/corecommon/crypto/stdeth.h" +#include "ypc/corecommon/crypto/gmssl.h" + +#include "convert_parser2.h" + +ypc::algo_wrapper> + pw; + +YPC_PARSER_IMPL(pw); diff --git a/example/convert/enclave2/user_type.h b/example/convert/enclave2/user_type.h new file mode 100644 index 00000000..8920839d --- /dev/null +++ b/example/convert/enclave2/user_type.h @@ -0,0 +1,7 @@ +#pragma once +#include "../common_t.h" +#include +#include +#include + +typedef row_t user_item_t; diff --git a/example/oram_personlist/CMakeLists.txt b/example/oram_personlist/CMakeLists.txt new file mode 100644 index 00000000..5ccc5851 --- /dev/null +++ b/example/oram_personlist/CMakeLists.txt @@ -0,0 +1,4 @@ +add_subdirectory(generator) +add_subdirectory(plugin) +add_subdirectory(first_match/enclave) +add_subdirectory(first_match/normal) diff --git a/example/oram_personlist/common.h b/example/oram_personlist/common.h new file mode 100644 index 00000000..1622d7d6 --- /dev/null +++ b/example/oram_personlist/common.h @@ -0,0 +1,7 @@ +#pragma once +#include "common_t.h" +#include "ypc/core/blockfile.h" +#include "ypc/core/byte.h" +#include "ypc/corecommon/package.h" +typedef ypc::blockfile<0x82, 1024 * 1024, 256 * 64 * 1024> file_t; + diff --git a/example/oram_personlist/common_t.h b/example/oram_personlist/common_t.h new file mode 100644 index 00000000..9cb25648 --- /dev/null +++ b/example/oram_personlist/common_t.h @@ -0,0 +1,36 @@ +#pragma once +#include +#include + +define_nt(RYXXBZ, std::string, "RYXXBZ"); +define_nt(XM, std::string, "XM"); +define_nt(CYM, std::string, "CYM"); +define_nt(XBDM, std::string, "XBDM"); +define_nt(FWXXBZ, std::string, "FWXXBZ"); +define_nt(XP, std::string, "XP"); +define_nt(DWMC, std::string, "DWMC"); +define_nt(ZJHM, std::string, "ZJHM"); +define_nt(GJDM, std::string, "GJDM"); +define_nt(MZDM, std::string, "MZDM"); +define_nt(JGSSXDM, std::string, "JGSSXDM"); +define_nt(HKXZFLYDM, std::string, "HKXZFLYDM"); +define_nt(HLXDM, std::string, "HLXDM"); +define_nt(HJDZ_XZQHDM, std::string, "HJDZXZQHDM"); +define_nt(SJJZD_XZQHDM, std::string, "SJJZDXZQHDM"); +define_nt(SJJZD_QHNXXDZ, std::string, "SJJZDQHNXXDZ"); +define_nt(XLDM, std::string, "XLDM"); +define_nt(TSSFDM, std::string, "TSSFDM"); +define_nt(CSQR, std::string, "CSQR"); +define_nt(LXDH, std::string, "LXDH"); +define_nt(HYZKDM, std::string, "HYZKDM"); +define_nt(DJR_XM, std::string, "DJR_XM"); +define_nt(DJR_GMSFZHM, std::string, "DJRGMSFZHM"); +define_nt(DJR_LXDH, std::string, "DJRLXDH"); +define_nt(GXSJ, std::string, "GXSJ"); +define_nt(SJZT, std::string, "SJZT"); + +typedef ff::util::ntobject< + RYXXBZ, XM, CYM, XBDM, FWXXBZ, XP, DWMC, ZJHM, GJDM, MZDM, JGSSXDM, + HKXZFLYDM, HLXDM, HJDZ_XZQHDM, SJJZD_XZQHDM, SJJZD_QHNXXDZ, XLDM, TSSFDM, + CSQR, LXDH, HYZKDM, DJR_XM, DJR_GMSFZHM, DJR_LXDH, GXSJ, SJZT> + row_t; diff --git a/example/oram_personlist/first_match/enclave/CMakeLists.txt b/example/oram_personlist/first_match/enclave/CMakeLists.txt new file mode 100644 index 00000000..be75bb14 --- /dev/null +++ b/example/oram_personlist/first_match/enclave/CMakeLists.txt @@ -0,0 +1,13 @@ +set(T_SRCS eparser.cpp) + +add_ypc_applet(person_first_match_oram + CRYPTO stdeth + SRCS ${T_SRCS}) + +if(SGX_MODE STREQUAL "Debug") + enclave_sign(person_first_match_oram KEY enclave_private.pem + CONFIG enclave.config.debug.xml) +else() + enclave_sign(person_first_match_oram KEY enclave_private.pem + CONFIG enclave.config.xml) +endif() diff --git a/example/oram_personlist/first_match/enclave/enclave.config.debug.xml b/example/oram_personlist/first_match/enclave/enclave.config.debug.xml new file mode 100644 index 00000000..471a1132 --- /dev/null +++ b/example/oram_personlist/first_match/enclave/enclave.config.debug.xml @@ -0,0 +1,12 @@ + + 0 + 0 + 0x50000 + 0x5000000 + 10 + 1 + + 0 + 0 + 0xFFFFFFFF + diff --git a/example/oram_personlist/first_match/enclave/enclave.config.xml b/example/oram_personlist/first_match/enclave/enclave.config.xml new file mode 100644 index 00000000..df005eb0 --- /dev/null +++ b/example/oram_personlist/first_match/enclave/enclave.config.xml @@ -0,0 +1,12 @@ + + 0 + 0 + 0x50000 + 0x5000000 + 10 + 1 + + 1 + 0 + 0xFFFFFFFF + diff --git a/example/oram_personlist/first_match/enclave/enclave.lds b/example/oram_personlist/first_match/enclave/enclave.lds new file mode 100644 index 00000000..0d5614f5 --- /dev/null +++ b/example/oram_personlist/first_match/enclave/enclave.lds @@ -0,0 +1,11 @@ +enclave.so +{ + global: + g_global_data_sim; + g_global_data; + enclave_entry; + g_peak_heap_used; + g_peak_rsrv_mem_committed; + local: + *; +}; diff --git a/example/oram_personlist/first_match/enclave/enclave_debug.lds b/example/oram_personlist/first_match/enclave/enclave_debug.lds new file mode 100644 index 00000000..0d5614f5 --- /dev/null +++ b/example/oram_personlist/first_match/enclave/enclave_debug.lds @@ -0,0 +1,11 @@ +enclave.so +{ + global: + g_global_data_sim; + g_global_data; + enclave_entry; + g_peak_heap_used; + g_peak_rsrv_mem_committed; + local: + *; +}; diff --git a/example/oram_personlist/first_match/enclave/enclave_private.pem b/example/oram_personlist/first_match/enclave/enclave_private.pem new file mode 100644 index 00000000..529d07be --- /dev/null +++ b/example/oram_personlist/first_match/enclave/enclave_private.pem @@ -0,0 +1,39 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIG4gIBAAKCAYEAroOogvsj/fZDZY8XFdkl6dJmky0lRvnWMmpeH41Bla6U1qLZ +AmZuyIF+mQC/cgojIsrBMzBxb1kKqzATF4+XwPwgKz7fmiddmHyYz2WDJfAjIveJ +ZjdMjM4+EytGlkkJ52T8V8ds0/L2qKexJ+NBLxkeQLfV8n1mIk7zX7jguwbCG1Pr +nEMdJ3Sew20vnje+RsngAzdPChoJpVsWi/K7cettX/tbnre1DL02GXc5qJoQYk7b +3zkmhz31TgFrd9VVtmUGyFXAysuSAb3EN+5VnHGr0xKkeg8utErea2FNtNIgua8H +ONfm9Eiyaav1SVKzPHlyqLtcdxH3I8Wg7yqMsaprZ1n5A1v/levxnL8+It02KseD +5HqV4rf/cImSlCt3lpRg8U5E1pyFQ2IVEC/XTDMiI3c+AR+w2jSRB3Bwn9zJtFlW +KHG3m1xGI4ck+Lci1JvWWLXQagQSPtZTsubxTQNx1gsgZhgv1JHVZMdbVlAbbRMC +1nSuJNl7KPAS/VfzAgEDAoIBgHRXxaynbVP5gkO0ug6Qw/E27wzIw4SmjsxG6Wpe +K7kfDeRskKxESdsA/xCrKkwGwhcx1iIgS5+Qscd1Yg+1D9X9asd/P7waPmWoZd+Z +AhlKwhdPsO7PiF3e1AzHhGQwsUTt/Y/aSI1MpHBvy2/s1h9mFCslOUxTmWw0oj/Q +ldIEgWeNR72CE2+jFIJIyml6ftnb6qzPiga8Bm48ubKh0kvySOqnkmnPzgh+JBD6 +JnBmtZbfPT97bwTT+N6rnPqOOApvfHPf15kWI8yDbprG1l4OCUaIUH1AszxLd826 +5IPM+8gINLRDP1MA6azECPjTyHXhtnSIBZCyWSVkc05vYmNXYUNiXWMajcxW9M02 +wKzFELO8NCEAkaTPxwo4SCyIjUxiK1LbQ9h8PSy4c1+gGP4LAMR8xqP4QKg6zdu9 +osUGG/xRe/uufgTBFkcjqBHtK5L5VI0jeNIUAgW/6iNbYXjBMJ0GfauLs+g1VsOm +WfdgXzsb9DYdMa0OXXHypmV4GwKBwQDUwQj8RKJ6c8cT4vcWCoJvJF00+RFL+P3i +Gx2DLERxRrDa8AVGfqaCjsR+3vLgG8V/py+z+dxZYSqeB80Qeo6PDITcRKoeAYh9 +xlT3LJOS+k1cJcEmlbbO2IjLkTmzSwa80fWexKu8/Xv6vv15gpqYl1ngYoqJM3pd +vzmTIOi7MKSZ0WmEQavrZj8zK4endE3v0eAEeQ55j1GImbypSf7Idh7wOXtjZ7WD +Dg6yWDrri+AP/L3gClMj8wsAxMV4ZR8CgcEA0fzDHkFa6raVOxWnObmRoDhAtE0a +cjUj976NM5yyfdf2MrKy4/RhdTiPZ6b08/lBC/+xRfV3xKVGzacm6QjqjZrUpgHC +0LKiZaMtccCJjLtPwQd0jGQEnKfMFaPsnhOc5y8qVkCzVOSthY5qhz0XNotHHFmJ +gffVgB0iqrMTvSL7IA2yqqpOqNRlhaYhNl8TiFP3gIeMtVa9rZy31JPgT2uJ+kfo +gV7sdTPEjPWZd7OshGxWpT6QfVDj/T9T7L6tAoHBAI3WBf2DFvxNL2KXT2QHAZ9t +k3imC4f7U+wSE6zILaDZyzygA4RUbwG0gv8/TJVn2P/Eynf76DuWHGlaiLWnCbSz +Az2DHBQBBaku409zDQym3j1ugMRjzzSQWzJg0SIyBH3hTmnYcn3+Uqcp/lEBvGW6 +O+rsXFt3pukqJmIV8HzLGGaLm62BHUeZf3dyWm+i3p/hQAL7Xvu04QW70xuGqdr5 +afV7p5eaeQIJXyGQJ0eylV/90+qxjMKiB1XYg6WYvwKBwQCL/ddpgOdHJGN8uRom +e7Zq0Csi3hGheMKlKbN3vcxT5U7MdyHtTZZOJbTvxKNNUNYH/8uD+PqDGNneb29G +BfGzvI3EASyLIcGZF3OhKwZd0jUrWk2y7Vhob91jwp2+t73vdMbkKyI4mHOuXvGv +fg95si9oO7EBT+Oqvhccd2J+F1IVXncccYnF4u5ZGWt5lLewN/pVr7MjjykeaHqN +t+rfnQam2psA6fL4zS2zTmZPzR2tnY8Y1GBTi0Ko1OKd1HMCgcAb5cB/7/AQlhP9 +yQa04PLH9ygQkKKptZp7dy5WcWRx0K/hAHRoi2aw1wZqfm7VBNu2SLcs90kCCCxp +6C5sfJi6b8NpNbIPC+sc9wsFr7pGo9SFzQ78UlcWYK2Gu2FxlMjonhka5hvo4zvg +WxlpXKEkaFt3gLd92m/dMqBrHfafH7VwOJY2zT3WIpjwuk0ZzmRg5p0pG/svVQEH +NZmwRwlopysbR69B/n1nefJ84UO50fLh5s5Zr3gBRwbWNZyzhXk= +-----END RSA PRIVATE KEY----- diff --git a/example/oram_personlist/first_match/enclave/eparser.cpp b/example/oram_personlist/first_match/enclave/eparser.cpp new file mode 100644 index 00000000..a683f7d0 --- /dev/null +++ b/example/oram_personlist/first_match/enclave/eparser.cpp @@ -0,0 +1,11 @@ +#include "first_match_parser.h" +#include "ypc/core_t/analyzer/algo_wrapper.h" +#include "ypc/core_t/analyzer/macro.h" +#include "ypc/corecommon/crypto/stdeth.h" + +ypc::algo_wrapper> + pw; + +YPC_PARSER_IMPL(pw); diff --git a/example/oram_personlist/first_match/enclave/first_match_parser.h b/example/oram_personlist/first_match/enclave/first_match_parser.h new file mode 100644 index 00000000..bce40a8e --- /dev/null +++ b/example/oram_personlist/first_match/enclave/first_match_parser.h @@ -0,0 +1,71 @@ +#include "ypc/corecommon/package.h" +#include "ypc/stbox/ebyte.h" +#include "ypc/stbox/stx_common.h" +#ifdef EXAMPLE_FM_NORMAL +#include +typedef ypc::bytes bytes; +#else +#include "ypc/core_t/analyzer/data_source.h" +#include "ypc/stbox/tsgx/log.h" +typedef stbox::bytes bytes; +#endif +#include "user_type.h" +#include "ypc/corecommon/data_source.h" +#include "ypc/corecommon/to_type.h" +#include +#include +#include +#include + +define_nt(input_buf, std::string); +typedef ff::net::ntpackage<0, input_buf> input_buf_t; + +class first_match_parser { +public: + first_match_parser() {} + first_match_parser(ypc::data_source *source) : m_source(source){}; + + inline bytes do_parse(const bytes ¶m) { + LOG(INFO) << "do parse"; + ypc::to_type converter(m_source); + // param must be serialized ntpackage + auto pkg = ypc::make_package::from_bytes(param); + int counter = 0; + hpda::processor::internal::filter_impl match( + &converter, [&](const user_item_t &v) { + counter++; + std::string zjhm = v.get(); + if (zjhm == pkg.get()) { + return true; + } + return false; + }); + + hpda::output::internal::memory_output_impl mo(&match); + mo.get_engine()->run(); + LOG(INFO) << "do parse done"; + + bytes result; + for (auto it : mo.values()) { + stbox::printf("found\n"); + result += it.get(); + result += " : "; + result += it.get(); + result += " ."; + } + return result; + } + + inline bool merge_parse_result(const std::vector &block_result, + const bytes ¶m, bytes &result) { + bytes s; + for (auto k : block_result) { + s = s + k; + } + result = s; + return false; + } + +protected: + ypc::data_source *m_source; +}; diff --git a/example/oram_personlist/first_match/enclave/user_type.h b/example/oram_personlist/first_match/enclave/user_type.h new file mode 100644 index 00000000..51778432 --- /dev/null +++ b/example/oram_personlist/first_match/enclave/user_type.h @@ -0,0 +1,7 @@ +#pragma once +#include "../../common_t.h" +#include +#include +#include + +typedef row_t user_item_t; diff --git a/example/oram_personlist/first_match/normal/CMakeLists.txt b/example/oram_personlist/first_match/normal/CMakeLists.txt new file mode 100644 index 00000000..b2eee5c1 --- /dev/null +++ b/example/oram_personlist/first_match/normal/CMakeLists.txt @@ -0,0 +1,3 @@ +# add_executable(find_person main.cpp) +# target_link_libraries(find_person core stbox_common ff_net hpda) +# target_link_directories(find_person PUBLIC ${FF_LIB_DIR}) diff --git a/example/oram_personlist/first_match/normal/main.cpp b/example/oram_personlist/first_match/normal/main.cpp new file mode 100644 index 00000000..16f79e2c --- /dev/null +++ b/example/oram_personlist/first_match/normal/main.cpp @@ -0,0 +1,18 @@ +#include "../../common.h" +#include "ypc/common/ignore.h" +#include "ypc/core/block_data_source.h" +#include "ypc/stbox/ebyte.h" +#define EXAMPLE_FM_NORMAL +#include "../enclave/first_match_parser.h" +#undef EXAMPLE_FM_NORMAL + +int main(int argc, char *argv[]) { + std::string fp = "person_list"; + ypc::bytes param("421003198607263380"); + file_t ssf; + ssf.open_for_read(fp.c_str()); + ypc::block_data_source sds(&ssf); + first_match_parser fmp(&sds); + fmp.do_parse(param); + return 0; +} diff --git a/example/oram_personlist/generator/CMakeLists.txt b/example/oram_personlist/generator/CMakeLists.txt new file mode 100644 index 00000000..19a1fd28 --- /dev/null +++ b/example/oram_personlist/generator/CMakeLists.txt @@ -0,0 +1,6 @@ + + +add_executable(personlist_gen_oram main.cpp) + +target_link_libraries(personlist_gen_oram ff_net core) +target_link_directories(personlist_gen_oram PUBLIC ${FF_LIB_DIR}) diff --git a/example/oram_personlist/generator/main.cpp b/example/oram_personlist/generator/main.cpp new file mode 100644 index 00000000..79e822c5 --- /dev/null +++ b/example/oram_personlist/generator/main.cpp @@ -0,0 +1,81 @@ +#include "../common.h" +#include "ypc/core/byte.h" +#include "ypc/corecommon/package.h" +#include +#include + +// typedef ff::sql::table, person_list_meta, +// RYXXBZ, XM, CYM, XBDM, FWXXBZ, XP, DWMC, ZJHM, GJDM, +// MZDM, JGSSXDM, HKXZFLYDM, HLXDM, HJDZ_XZQHDM, +// SJJZD_XZQHDM, SJJZD_QHNXXDZ, XLDM, TSSFDM, CSQR, LXDH, +// HYZKDM, DJR_XM, DJR_GMSFZHM, DJR_LXDH, GXSJ, SJZT> +// person_list_table_t; + +// typedef typename person_list_table_t::row_collection_type::row_type row_t; + +row_t create(const std::string &id) { + row_t t; + + t.set("4028f c 856517466 f 0165174f 03680103"); + t.set("张三"); + // t.set("张三+" + id); + t.set("4028f c856517466 f 0165174f 03680105"); + t.set(id); + t.set("156"); + t.set("01"); + t.set("01"); + t.set("420500"); + t.set("10"); + t.set("02"); + t.set("420500"); + t.set("XX省直昌市城申半岛"); + t.set("420500"); + t.set("XX省宜昌市域中 半岛"); + t.set("20"); + t.set(id); + t.set("13800000000"); + t.set("10"); + t.set("张三"); + t.set("13800000000"); + t.set("20200701170500"); + + return t; +} + +void write_to_file(const std::string &path, int num) { + uint64_t id = 421003198607262336; + file_t f; + f.open_for_write(path.c_str()); + + for (int i = 0; i < num; i++) { + row_t t = create(std::to_string(id + i)); + typename ypc::cast_obj_to_package::type pt = t; + auto buf = ypc::make_bytes::for_package(pt); + f.append_item(buf.data(), buf.size()); + } + f.close(); +} + +void check_file(const std::string &path) { + + file_t f; + f.open_for_read(path.c_str()); + ypc::memref r; + uint64_t id = 421003198607262336; + int i = 0; + while (f.next_item(r)) { + typedef typename ypc::cast_obj_to_package::type pkg_t; + auto pkg = + ypc::make_package::from_bytes(ypc::bytes(r.data(), r.size())); + + // std::cout << pkg.get() << std::endl; + i++; + } + std::cout << "checked " << i << " items" << std::endl; +} + +int main(int argc, char *argv[]) { + write_to_file("person_list", 7970); + check_file("person_list"); + return 0; +} diff --git a/example/oram_personlist/plugin/CMakeLists.txt b/example/oram_personlist/plugin/CMakeLists.txt new file mode 100644 index 00000000..2eca53fd --- /dev/null +++ b/example/oram_personlist/plugin/CMakeLists.txt @@ -0,0 +1,5 @@ +add_library(person_reader_oram + SHARED + person_reader_oram.cpp) + +target_link_libraries(person_reader_oram core) diff --git a/example/oram_personlist/plugin/person_reader_oram.cpp b/example/oram_personlist/plugin/person_reader_oram.cpp new file mode 100644 index 00000000..6745a238 --- /dev/null +++ b/example/oram_personlist/plugin/person_reader_oram.cpp @@ -0,0 +1,89 @@ +#include "person_reader_oram.h" +#include "ypc/common/limits.h" +#include "../common.h" +#include +#include +#include +#include +#include + +void *create_item_reader(const char *file_path, int len) { + + file_t *f = new file_t(); + f->open_for_read(file_path); + f->reset_read_item(); + return f; +} + +int reset_for_read(void *handle) { + if (!handle) { + return -1; + } + file_t *f = (file_t *)handle; + f->reset_read_item(); + return 0; +} + +int read_item_data(void *handle, char *buf, int *len) { + if (!handle) { + return -1; + } + if (!buf) { + return -1; + } + file_t *f = (file_t *)handle; + + ypc::memref r; + bool t = f->next_item(r); + if (t) { + memcpy(buf, r.data(), r.size()); + *len = r.size(); + r.dealloc(); + return 0; + } else { + *len = 0; + return -1; + } + + return 0; +} + +int close_item_reader(void *handle) { + file_t *f = (file_t *)handle; + f->close(); + return 0; +} + +uint64_t get_item_number(void *handle) { + file_t *f = (file_t *)handle; + return f->item_number(); +} + +int get_item_index_field(void *handle, char *buf, int *len) { + if (!handle) { + return -1; + } + if (!buf) { + return -1; + } + file_t *f = (file_t *)handle; + + ypc::memref r; + bool t = f->next_item(r); + if (t) { + typedef typename ypc::cast_obj_to_package::type pkg_t; + auto item_pkg = + ypc::make_package::from_bytes(ypc::bytes(r.data(), r.size())); + std::string index_field = item_pkg.get(); + memcpy(buf, index_field.c_str(), index_field.size()); + *len = index_field.size(); + + r.dealloc(); + return 0; + } else { + *len = 0; + return -1; + } + + return 0; +} \ No newline at end of file diff --git a/example/oram_personlist/plugin/person_reader_oram.h b/example/oram_personlist/plugin/person_reader_oram.h new file mode 100644 index 00000000..20dd98be --- /dev/null +++ b/example/oram_personlist/plugin/person_reader_oram.h @@ -0,0 +1,18 @@ +#pragma once +#include +#ifdef __cplusplus +extern "C" { +#endif + +void *create_item_reader(const char *file_path, int len); + +int reset_for_read(void *handle); +int read_item_data(void *handle, char *buf, int *len); +int close_item_reader(void *handle); +uint64_t get_item_number(void *handle); + +int get_item_index_field(void *handle, char *buf, int *len); + +#ifdef __cplusplus +} +#endif diff --git a/hpda/test/CMakeLists.txt b/hpda/test/CMakeLists.txt index a9c6e3ba..da41565a 100644 --- a/hpda/test/CMakeLists.txt +++ b/hpda/test/CMakeLists.txt @@ -10,6 +10,8 @@ add_executable(hpda_gtest gtest_main.cpp processor/test_concat.cpp processor/test_split.cpp processor/test_set.cpp + processor/test_trim.cpp + processor/test_all.cpp kmeans/test_loyd_vector.cpp kmeans/test_loyd_tuple.cpp kmeans/test_kmeans_tuple.cpp diff --git a/hpda/test/processor/test_all.cpp b/hpda/test/processor/test_all.cpp new file mode 100644 index 00000000..ea7cfda0 --- /dev/null +++ b/hpda/test/processor/test_all.cpp @@ -0,0 +1,30 @@ +#include +#include + +define_nt(hash, std::string); +define_nt(value, uint64_t); +typedef hpda::ntobject data_item_t; + +TEST(all, basic) { + hpda::extractor::raw_data rd; + + hpda::engine engine; + rd.set_engine(&engine); + + for (int i = 0; i < 1000; i++) { + data_item_t d; + d.set(std::to_string(i), i); + rd.add_data(d); + } + + hpda::processor::all_t f(&rd); + hpda::output::memory_output mo(&f); + + engine.run(); + auto &s = f.values(); + EXPECT_EQ(s.size(), 1000); + + for (int i = 0; i < 1000; i++) { + EXPECT_EQ(s[i].get<::value>(), i); + } +} diff --git a/hpda/test/processor/test_trim.cpp b/hpda/test/processor/test_trim.cpp new file mode 100644 index 00000000..d85e1b34 --- /dev/null +++ b/hpda/test/processor/test_trim.cpp @@ -0,0 +1,31 @@ +#include +#include + +define_nt(hash, std::string); +define_nt(value, uint64_t); +typedef hpda::ntobject data_item_t; +typedef hpda::ntobject value_item_t; + +TEST(trim, basic) { + hpda::extractor::raw_data rd; + + hpda::engine engine; + rd.set_engine(&engine); + + for (int i = 0; i < 1000; i++) { + data_item_t d; + d.set(std::to_string(i), i); + rd.add_data(d); + } + + hpda::processor::trim_t f(&rd); + hpda::output::memory_output mo(&f); + + engine.run(); + auto &s = mo.values(); + EXPECT_EQ(s.size(), 1000); + + for (int i = 0; i < 1000; i++) { + EXPECT_EQ(s[i].get<::value>(), i); + } +} diff --git a/include/hpda/processor/processor.h b/include/hpda/processor/processor.h index 1cf85051..71f15445 100644 --- a/include/hpda/processor/processor.h +++ b/include/hpda/processor/processor.h @@ -4,7 +4,9 @@ #include #include #include +#include #include #include #include #include +#include diff --git a/include/hpda/processor/transform/all.h b/include/hpda/processor/transform/all.h new file mode 100644 index 00000000..684bb27b --- /dev/null +++ b/include/hpda/processor/transform/all.h @@ -0,0 +1,38 @@ +#pragma once +#include + +namespace hpda { +namespace processor { +namespace internal { + +template +class all_impl : public processor_base { +public: + all_impl(::hpda::internal::processor_with_output *upper_stream) + : processor_base(upper_stream) {} + + virtual ~all_impl() {} + + typedef processor_base base; + + virtual bool process() { + if (!base::has_input_value()) { + return false; + } + m_data.push_back(base::input_value().make_copy()); + base::consume_input_value(); + return true; + } + + virtual InputObjType output_value() { return m_data.back(); } + + std::vector &values() { return m_data; } + const std::vector &values() const { return m_data; } + +protected: + std::vector m_data; +}; +} // namespace internal +template using all_t = internal::all_impl; +} // namespace processor +} // namespace hpda diff --git a/include/hpda/processor/transform/trim.h b/include/hpda/processor/transform/trim.h new file mode 100644 index 00000000..531314e6 --- /dev/null +++ b/include/hpda/processor/transform/trim.h @@ -0,0 +1,35 @@ +#pragma once +#include + +namespace hpda { +namespace processor { +namespace internal { + +template +class trim_impl : public processor_base { +public: + trim_impl(::hpda::internal::processor_with_output *upper_stream) + : processor_base(upper_stream) {} + + virtual ~trim_impl() {} + + typedef processor_base base; + + virtual bool process() { + if (!base::has_input_value()) { + return false; + } + m_data = base::input_value().make_copy(); + base::consume_input_value(); + return true; + } + + virtual OutputObjType output_value() { return m_data; } + +protected: + OutputObjType m_data; +}; +} // namespace internal +template using trim_t = internal::trim_impl; +} // namespace processor +} // namespace hpda diff --git a/include/ypc/common/parser_type.h b/include/ypc/common/parser_type.h index 2df5004b..999f169b 100644 --- a/include/ypc/common/parser_type.h +++ b/include/ypc/common/parser_type.h @@ -17,6 +17,7 @@ constexpr static uint32_t single_sealed_datasource_parser = 1; constexpr static uint32_t multi_sealed_datasource_parser = 2; constexpr static uint32_t noinput_datasource_parser = 3; constexpr static uint32_t raw_datasource_parser = 4; +constexpr static uint32_t oram_sealed_datasource_parser = 5; constexpr static uint32_t no_model_parser = 0; constexpr static uint32_t has_model_parser = 1; diff --git a/include/ypc/core/oram_sealed_file.h b/include/ypc/core/oram_sealed_file.h new file mode 100644 index 00000000..6214ffc2 --- /dev/null +++ b/include/ypc/core/oram_sealed_file.h @@ -0,0 +1,81 @@ +#pragma once +#include "ypc/core/oramblockfile.h" +#include "ypc/common/limits.h" +#include "ypc/core/byte.h" + + +namespace ypc { +namespace internal { + class oram_sealed_file_base { +public: + using oramblockfile_t = ypc::oram::oramblockfile<1024 * 1024 * 256>; + + oram_sealed_file_base(const std::string &file_path); + + virtual ~oram_sealed_file_base() = default; + + virtual void reset() = 0; + + virtual void open_for_write() = 0; + + virtual void open_for_read() = 0; + + virtual bool download_oram_params(uint32_t *block_num, uint32_t *bucket_num_N, + uint8_t *level_num_L, uint32_t *bucket_str_size, uint32_t *batch_str_size) = 0; + + virtual bool get_block_id(const bytes &item_index_field_hash, uint32_t *block_id) = 0; + + virtual bool download_position_map(memref &posmap) = 0; + + virtual bool update_position_map(const uint8_t * position_map, uint32_t len) = 0; + + virtual bool download_path(uint32_t leaf, memref &en_path) = 0; + + virtual bool upload_path(uint32_t leaf, const uint8_t * encrpypted_path, uint32_t len) = 0; + + virtual bool download_stash(memref &st) = 0; + + virtual bool update_stash(const uint8_t * stash, uint32_t len) = 0; + + virtual bool read_root_hash(memref &root_hash) = 0; + + virtual bool download_merkle_hash(uint32_t leaf, memref &merkle_hash) = 0; + + virtual bool update_merkle_hash(uint32_t leaf, const uint8_t * merkle_hash, uint32_t len) = 0; + + +public: + oram_sealed_file_base(const oram_sealed_file_base &) = delete; + oram_sealed_file_base(oram_sealed_file_base &&) = delete; + oram_sealed_file_base &operator=(oram_sealed_file_base &&) = delete; + oram_sealed_file_base &operator=(const oram_sealed_file_base &) = delete; + +protected: + oramblockfile_t m_file; +}; +} // namespace internal + +class simple_oram_sealed_file : public internal::oram_sealed_file_base { +public: + simple_oram_sealed_file(const std::string &file_path); + virtual void reset(); + virtual void open_for_write(); + virtual void open_for_read(); + virtual bool download_oram_params(uint32_t *block_num, uint32_t *bucket_num_N, + uint8_t *level_num_L, uint32_t *bucket_str_size, uint32_t *batch_str_size); + virtual bool get_block_id(const bytes &item_index_field_hash, uint32_t *block_id); + virtual bool download_position_map(memref &posmap); + virtual bool update_position_map(const uint8_t * position_map, uint32_t len); + virtual bool download_path(uint32_t leaf, memref &en_path); + virtual bool upload_path(uint32_t leaf, const uint8_t * encrpypted_path, uint32_t len); + virtual bool download_stash(memref &st); + virtual bool update_stash(const uint8_t * stash, uint32_t len); + // virtual bool read_root_hash(bytes &root_hash); + virtual bool read_root_hash(memref &root_hash); + virtual bool download_merkle_hash(uint32_t leaf, memref &merkle_hash); + virtual bool update_merkle_hash(uint32_t leaf, const uint8_t * merkle_hash, uint32_t len); +}; + + + +} \ No newline at end of file diff --git a/include/ypc/core/oramblockfile.h b/include/ypc/core/oramblockfile.h new file mode 100644 index 00000000..90424373 --- /dev/null +++ b/include/ypc/core/oramblockfile.h @@ -0,0 +1,381 @@ +#pragma once +#include "ypc/corecommon/oram_types.h" +#include "ypc/core/exceptions.h" +#include "ypc/core/memref.h" +#include "ypc/core/byte.h" +#include "ypc/corecommon/package.h" + +#include +#include +#include +#include +#include + +using oram_ntt = ypc::oram::nt; + +namespace ypc { +namespace oram { + +template + // uint32_t OramBlockSizeLimit_t, uint8_t OramBucketSize_t> +class oramblockfile { +public: + const static uint64_t BlockNumLimit = OramBlockNumLimit_t; + // const static uint32_t DataSizeB = OramBlockSizeLimit_t; + // const static uint8_t BucketSizeZ = OramBucketSize_t; + const static uint32_t hash_size = 32; + + oramblockfile(const std::string &file_path): m_file(), m_file_path(file_path), m_header(), m_id_map() { + // open_for_write(); + } + + oramblockfile(const oramblockfile &) = delete; + oramblockfile(oramblockfile &&) = delete; + oramblockfile &operator=(const oramblockfile &) = delete; + oramblockfile &operator=(oramblockfile &&) = delete; + virtual ~oramblockfile() = default; + + void open_for_read() { + if(m_file.is_open()) { + throw std::runtime_error("already open"); + } + + m_file.open(m_file_path, std::ios::in | std::ios::binary); + if (!m_file.is_open()) { + throw file_open_failure(m_file_path, "oramblockfile::open_for_read"); + } + + read_header(); + } + + void open_for_write() { + if (m_file.is_open()) { + throw std::runtime_error("already open"); + } + + m_file.open(m_file_path, std::ios::in | std::ios::out | std::ios::binary); + if (!m_file.is_open()) { + throw file_open_failure(m_file_path, "oramblockfile::open_for_write"); + } + read_header(); + } + + void reset() { + m_file.clear(); + read_header(); + } + + void read_header() { + m_file.seekg(0, m_file.beg); + m_file.read((char *)&m_header, sizeof(m_header)); + } + + void read_id_map() { + m_file.seekg(m_header.id_map_filepos, m_file.beg); + bytes id_map_str(m_header.position_map_filepos - m_header.id_map_filepos); + m_file.read((char*)id_map_str.data(), id_map_str.size()); + + auto id_map_pkg = make_package::from_bytes(id_map_str); + auto id_map_array = id_map_pkg.get(); + + for(const auto& element : id_map_array) { + bytes item_index_field_hash = element.get(); + uint32_t block_id = element.get(); + m_id_map.insert({item_index_field_hash, block_id}); + } + + } + + + bool download_position_map(memref &posmap) { + size_t len = m_header.merkle_tree_filepos - m_header.position_map_filepos; + if(posmap.data() == nullptr) { + posmap.alloc(len); + } + if(posmap.size() < len) { + posmap.dealloc(); + posmap.alloc(len); + } + + m_file.seekg(m_header.position_map_filepos, m_file.beg); + m_file.read((char *)posmap.data(), len); + posmap.size() = len; + return true; + } + + bool get_block_id(const bytes &item_index_field_hash, uint32_t *block_id) { + read_id_map(); + if(m_id_map.find(item_index_field_hash) == m_id_map.end()) { + return false; + } + *block_id = m_id_map.at(item_index_field_hash); + return true; + } + + const uint32_t& get_block_num() const{ + return m_header.block_num; + } + + const uint32_t& get_bucket_num() const{ + return m_header.bucket_num_N; + } + + const uint8_t& get_level_num() const{ + return m_header.level_num_L; + } + + const uint32_t& get_bucket_str_size() const{ + return m_header.bucket_str_size; + } + + const uint32_t& get_batch_str_size() const{ + return m_header.batch_str_size; + } + + bool update_position_map(const uint8_t * position_map, uint32_t len) { + read_header(); + m_file.clear(); + + m_file.seekp(m_header.position_map_filepos, m_file.beg); + m_file.write((char *)(position_map), len); + return true; + } + + + bool download_path(uint32_t leaf, memref &en_path) { + std::vector offsets; + leaf_to_offsets(leaf, offsets); + + std::vector en_path_array; + for(const uint32_t& offset : offsets) { + bytes en_bucket_str(m_header.bucket_str_size); + m_file.seekg(m_header.oram_tree_filepos + offset * m_header.bucket_str_size, m_file.beg); + m_file.read((char *)en_bucket_str.data(), m_header.bucket_str_size); + en_path_array.push_back(en_bucket_str); + } + + oram_ntt::path_pkg_t path_pkg; + path_pkg.set(en_path_array); + bytes en_path_str = make_bytes::for_package(path_pkg); + + size_t len = en_path_str.size(); + if(en_path.data() == nullptr) { + en_path.alloc(len); + } + if(en_path.size() < len) { + en_path.dealloc(); + en_path.alloc(len); + } + + memcpy(en_path.data(), en_path_str.data(), len); + en_path.size() = len; + + return true; + } + + bool upload_path(uint32_t leaf, const uint8_t * encrpypted_path, uint32_t len) { + read_header(); + m_file.clear(); + + bytes encrpypted_path_str(len); + memcpy(encrpypted_path_str.data(), encrpypted_path, len); + + oram_ntt::path_pkg_t path_pkg = make_package::from_bytes(encrpypted_path_str); + std::vector bucket_str_array = path_pkg.get(); + + std::vector offsets; + leaf_to_offsets(leaf, offsets); + for(uint8_t i = 0; i < offsets.size(); ++i) { + m_file.seekp(m_header.oram_tree_filepos + offsets[i] * m_header.bucket_str_size, m_file.beg); + m_file.write((char *)bucket_str_array[i].data(), m_header.bucket_str_size); + } + + return true; + } + + bool download_stash(memref &st) { + if(m_header.stash_size == 0) { + return true; + } + + size_t len = m_header.stash_size; + if(st.data() == nullptr) { + st.alloc(len); + } + if(st.size() < len) { + st.dealloc(); + st.alloc(len); + } + + m_file.seekg(m_header.stash_filepos, m_file.beg); + m_file.read((char *)st.data(), len); + st.size() = len; + return true; + } + + bool update_stash(const uint8_t * stash, uint32_t len) { + // read_header(); + // m_file.clear(); + + m_header.stash_size = len; + m_file.seekp(0, m_file.beg); + m_file.write((char *)&m_header, sizeof(m_header)); + + if(len > 0) { + m_file.seekp(m_header.stash_filepos, m_file.beg); + m_file.write((char *)(stash), len); + } + + return true; + } + + void close() { + m_file.close(); + } + + bool read_root_hash(memref &root_hash) { + size_t len = hash_size; + if(root_hash.data() == nullptr) { + root_hash.alloc(len); + } + if(root_hash.size() < len) { + root_hash.dealloc(); + root_hash.alloc(len); + } + + m_file.seekg(m_header.merkle_tree_filepos, m_file.beg); + m_file.read((char *)root_hash.data(), len); + root_hash.size() = len; + + return true; + } + + bool download_merkle_hash(uint32_t leaf, memref &merkle_hash) { + std::vector hash_offsets; + get_hash_offsets(leaf, hash_offsets); + + std::vector path_offsets; + leaf_to_offsets(leaf, path_offsets); + + std::vector merkle_hash_array; + uint32_t i = 0; + for(const uint32_t& offset : hash_offsets) { + bytes data_hash(hash_size); + bool in_path = false; + m_file.seekg(m_header.merkle_tree_filepos + offset * hash_size, m_file.beg); + m_file.read((char *)data_hash.data(), hash_size); + + if(offset == path_offsets[i]) { + ++i; + in_path = true; + } + oram_ntt::hash_pair hash_p; + hash_p.set(data_hash, in_path); + merkle_hash_array.push_back(hash_p); + } + + oram_ntt::merkle_hash_pkg_t merkle_hash_pkg; + merkle_hash_pkg.set(merkle_hash_array); + bytes merkle_hash_str = make_bytes::for_package(merkle_hash_pkg); + + size_t len = merkle_hash_str.size(); + if(merkle_hash.data() == nullptr) { + merkle_hash.alloc(len); + } + if(merkle_hash.size() < len) { + merkle_hash.dealloc(); + merkle_hash.alloc(len); + } + + memcpy(merkle_hash.data(), merkle_hash_str.data(), len); + merkle_hash.size() = len; + + return true; + } + + bool update_merkle_hash(uint32_t leaf, const uint8_t * merkle_hash, uint32_t len) { + read_header(); + m_file.clear(); + + bytes merkle_hash_str(len); + memcpy(merkle_hash_str.data(), merkle_hash, len); + + oram_ntt::merkle_hash_pkg_t merkle_hash_pkg = + make_package::from_bytes(merkle_hash_str); + auto merkle_hash_array = merkle_hash_pkg.get(); + + std::vector offsets; + get_hash_offsets(leaf, offsets); + for(uint8_t i = 0; i < offsets.size(); ++i) { + m_file.seekp(m_header.merkle_tree_filepos + offsets[i] * hash_size, m_file.beg); + m_file.write((char *)merkle_hash_array[i].get().data(), hash_size); + } + + return true; + } + + +private: + + void get_hash_offsets(uint32_t leaf, std::vector &offsets) { + if(leaf == 0) { + throw std::runtime_error("leaf label is invalid!"); + } + + // Convert leaf to index in ORAM tree + uint32_t cur_node = (1 << m_header.level_num_L) - 1 + leaf - 1; + + bool flag = true; + while(flag) { + if(cur_node == 0) { + offsets.push_back(cur_node); + break; + } + + if(cur_node % 2 == 0) { // cur_node is a right child node + offsets.push_back(cur_node); + offsets.push_back(cur_node - 1); + } else { + offsets.push_back(cur_node + 1); + offsets.push_back(cur_node); + } + + cur_node = (cur_node - 1) / 2; + } + std::reverse(offsets.begin(), offsets.end()); + + } + + void leaf_to_offsets(uint32_t leaf, std::vector &offsets) { + if(leaf == 0) { + throw std::runtime_error("leaf label is invalid!"); + } + + // Convert leaf to index in ORAM tree + uint32_t cur_node = (1 << m_header.level_num_L) - 1 + leaf - 1; + + bool flag = true; + while(flag) { + if(cur_node == 0) + flag = false; + + offsets.push_back(cur_node); + + cur_node = (cur_node - 1) / 2; + } + std::reverse(offsets.begin(), offsets.end()); + } + + + std::fstream m_file; + std::string m_file_path; + header m_header; + std::unordered_map m_id_map; +}; + + + + +} + +} \ No newline at end of file diff --git a/include/ypc/core/privacy_data_reader.h b/include/ypc/core/privacy_data_reader.h index 586a6d3d..3b9649b0 100644 --- a/include/ypc/core/privacy_data_reader.h +++ b/include/ypc/core/privacy_data_reader.h @@ -43,6 +43,8 @@ class privacy_data_reader { bytes get_sample_data(); std::string get_data_format(); + std::string get_item_index_field(); + protected: template T get_func_with_name(const std::string &name) { T r = (T)dlsym(m_lib_handle, name.c_str()); @@ -65,6 +67,7 @@ class privacy_data_reader { typedef uint64_t (*get_item_number_func_t)(void *); // NOLINT typedef int (*get_sample_data_func_t)(void *, char *, int *); // NOLINT typedef int (*get_data_format_func_t)(void *, char *, int *); // NOLINT + typedef int (*get_item_index_field_func_t)(void *, char *, int *); // NOLINT // NOLINTEND(modernize-use-using) const std::string m_plugin_path; @@ -81,6 +84,8 @@ class privacy_data_reader { get_item_number_func_t m_get_item_number; get_sample_data_func_t m_get_sample_data; get_data_format_func_t m_get_data_format; + + get_item_index_field_func_t m_get_item_index_field; }; } // namespace ypc diff --git a/include/ypc/core_t/analyzer/algo_wrapper.h b/include/ypc/core_t/analyzer/algo_wrapper.h index 5570d0c9..a8951f05 100644 --- a/include/ypc/core_t/analyzer/algo_wrapper.h +++ b/include/ypc/core_t/analyzer/algo_wrapper.h @@ -18,6 +18,8 @@ #include "ypc/core_t/analyzer/internal/data_streams/noinput_data_stream.h" #include "ypc/core_t/analyzer/internal/data_streams/raw_data_stream.h" #include "ypc/core_t/analyzer/internal/data_streams/sealed_data_stream.h" +#include "ypc/core_t/analyzer/internal/data_streams/oram_sealed_data_stream.h" +#include "ypc/core_t/analyzer/internal/data_streams/convert_sealed_data_stream.h" #include "ypc/core_t/analyzer/internal/keymgr_session.h" #include "ypc/core_t/analyzer/internal/results/local_result.h" #include "ypc/core_t/analyzer/internal/results/offchain_result.h" diff --git a/include/ypc/core_t/analyzer/data_source.h b/include/ypc/core_t/analyzer/data_source.h index a1a9eed9..9be8256b 100644 --- a/include/ypc/core_t/analyzer/data_source.h +++ b/include/ypc/core_t/analyzer/data_source.h @@ -22,11 +22,26 @@ class data_source_with_dhash : public data_source { inline const bytes &expect_data_hash() const { return m_expect_data_hash; } virtual const bytes &data_hash() const = 0; - inline void reset_reach_end(){m_data_reach_end = false;} + inline void reset_reach_end() { m_data_reach_end = false; } protected: bytes m_expect_data_hash; bool m_data_reach_end; uint32_t m_counter; }; + +class data_source_with_merkle_hash : public data_source { +public: + data_source_with_merkle_hash(const stbox::bytes &data_hash) + : m_expect_root_hash(data_hash) {} + + virtual ~data_source_with_merkle_hash() {} + + inline const std::vector &expect_data_hash() const { return m_expect_data_hash; } + virtual const std::vector &data_hash() const = 0; + +protected: + bytes m_expect_root_hash; + std::vector m_expect_data_hash; +}; } // namespace ypc diff --git a/include/ypc/core_t/analyzer/eparser_t_interface.h b/include/ypc/core_t/analyzer/eparser_t_interface.h index e095537d..62de6487 100644 --- a/include/ypc/core_t/analyzer/eparser_t_interface.h +++ b/include/ypc/core_t/analyzer/eparser_t_interface.h @@ -35,6 +35,51 @@ sgx_status_t SGX_CDECL next_data_batch(uint32_t *retval, uint32_t hash_size, uint8_t **data, uint32_t *len); sgx_status_t SGX_CDECL free_data_batch(uint8_t *data); +sgx_status_t SGX_CDECL download_oram_params_OCALL(uint32_t *retval, const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_num, uint32_t *bucket_num_N, uint8_t *level_num_L, + uint32_t *bucket_str_size, uint32_t *batch_str_size); +sgx_status_t SGX_CDECL get_block_id_OCALL(uint32_t *retval, const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_id, + const uint8_t *param_hash, uint32_t param_hash_size); +sgx_status_t SGX_CDECL download_position_map_OCALL(uint32_t *retval, const uint8_t *data_hash, uint32_t hash_size, + uint8_t ** position_map, uint32_t *len); +sgx_status_t SGX_CDECL update_position_map_OCALL(uint32_t *retval, const uint8_t *data_hash, uint32_t hash_size, + uint8_t * position_map, uint32_t len); +sgx_status_t SGX_CDECL download_path_OCALL(uint32_t *retval, const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t ** encrpypted_path, uint32_t *len); +sgx_status_t SGX_CDECL download_stash_OCALL(uint32_t *retval, const uint8_t *data_hash, uint32_t hash_size, + uint8_t ** stash, uint32_t *len); +sgx_status_t SGX_CDECL update_stash_OCALL(uint32_t *retval, const uint8_t *data_hash, uint32_t hash_size, + uint8_t * stash, uint32_t len); +sgx_status_t SGX_CDECL upload_path_OCALL(uint32_t *retval, const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t * encrpypted_path, uint32_t len); +sgx_status_t SGX_CDECL download_merkle_hash_OCALL(uint32_t *retval, const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t ** merkle_hash, uint32_t *len); +sgx_status_t SGX_CDECL update_merkle_hash_OCALL(uint32_t *retval, const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t * merkle_hash, uint32_t len); + + + +sgx_status_t SGX_CDECL write_convert_data_structure(uint32_t *retval, int64_t filepos, + uint8_t * convert_data_bytes, uint32_t len); +sgx_status_t SGX_CDECL download_convert_params_ocall(uint32_t *retval, uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size); + + +sgx_status_t SGX_CDECL km_session_request_oram_ocall(uint32_t *retval, + sgx_dh_msg1_t *dh_msg1, + uint32_t *session_id); +sgx_status_t SGX_CDECL km_exchange_report_oram_ocall(uint32_t *retval, + sgx_dh_msg2_t *dh_msg2, + sgx_dh_msg3_t *dh_msg3, + uint32_t session_id); +sgx_status_t SGX_CDECL km_send_request_oram_ocall( + uint32_t *retval, uint32_t session_id, secure_message_t *req_message, + size_t req_message_size, size_t max_payload_size, + secure_message_t *resp_message, size_t resp_message_size); +sgx_status_t SGX_CDECL km_end_session_oram_ocall(uint32_t *retval, + uint32_t session_id); + sgx_status_t SGX_CDECL sgx_oc_cpuidex(int cpuinfo[4], int leaf, int subleaf); sgx_status_t SGX_CDECL sgx_thread_wait_untrusted_event_ocall(int *retval, const void *self); diff --git a/include/ypc/core_t/analyzer/interface/algo_interface.h b/include/ypc/core_t/analyzer/interface/algo_interface.h index cd97df7a..af95fd58 100644 --- a/include/ypc/core_t/analyzer/interface/algo_interface.h +++ b/include/ypc/core_t/analyzer/interface/algo_interface.h @@ -13,10 +13,13 @@ #include "ypc/corecommon/allowance.h" #include "ypc/stbox/ebyte.h" #include "ypc/stbox/stx_status.h" +#include "ypc/core_t/analyzer/internal/data_streams/convert_sealed_data_stream.h" +#include "ypc/corecommon/oram_types.h" namespace ypc { namespace internal { typedef nt ntt; +typedef oram::nt oram_ntt; template class DataAllowancePolicy, + template class ModelAllowancePolicy> +class algo_interface + : virtual public parser_var, + virtual public request_key_var, + virtual public result_var, + virtual public do_parse_interface, + virtual public keymgr_interface, + virtual public encrypted_param_var, + virtual public check_allowance_interface, + virtual public forward_interface { + typedef Crypto ecc; + typedef keymgr_interface keymgr_interface_t; + typedef request_key_var request_key_var_t; + typedef do_parse_interface do_parse_interface_t; + typedef check_allowance_interface + allowance_checker_t; + typedef forward_interface forward_interface_t; + +protected: + uint32_t parse_data_item_impl(const uint8_t *input_param, uint32_t len) { +#ifdef DEBUG + LOG(INFO) << "start parse data"; +#endif + ntt::param_t param = + make_package::type>::from_bytes( + input_param, len); + +#ifdef DEBUG + LOG(INFO) << "done unmarshal param, start request private key"; +#endif + + request_key_var_t::m_pkey4v = param.get(); + stbox::bytes dian_pkey; + auto ret = keymgr_interface_t::request_private_key_for_public_key( + request_key_var_t::m_pkey4v, request_key_var_t::m_private_key, + dian_pkey); + if (ret) { + LOG(ERROR) << "request_private_key failed: " << stbox::status_string(ret); + return ret; + } + +#ifdef DEBUG + LOG(INFO) << "start do_parse"; +#endif + result_var::m_result = do_parse_interface_t::do_parse( + request_key_var_t::m_pkey4v.data(), request_key_var_t::m_pkey4v.size()); + +#ifdef DEBUG + LOG(INFO) << "end parse "; +#endif + result_var::m_cost_gas = 0; + return stbox::stx_status::success; + } +}; + template class DataAllowancePolicy, diff --git a/include/ypc/core_t/analyzer/interface/data_hash_interface.h b/include/ypc/core_t/analyzer/interface/data_hash_interface.h index e8d07832..3b79c887 100644 --- a/include/ypc/core_t/analyzer/interface/data_hash_interface.h +++ b/include/ypc/core_t/analyzer/interface/data_hash_interface.h @@ -57,5 +57,20 @@ class data_hash_interface } }; +template +class data_hash_interface + : virtual public data_merkle_hash_var, + virtual public data_source_var { + typedef data_source_var data_source_var_t; + +protected: + void set_data_hash() { + std::vector data_hash_array = data_source_var_t::m_datasource->data_hash(); + for(uint32_t i = 0; i < data_hash_array.size(); ++i) { + data_merkle_hash_var::m_data_hash.push_back(data_hash_array[i]); + } + } +}; + } // namespace internal } // namespace ypc diff --git a/include/ypc/core_t/analyzer/interface/data_interface.h b/include/ypc/core_t/analyzer/interface/data_interface.h index 92f522e7..8929861b 100644 --- a/include/ypc/core_t/analyzer/interface/data_interface.h +++ b/include/ypc/core_t/analyzer/interface/data_interface.h @@ -1,17 +1,21 @@ -#pragma oncew +#pragma once #include "ypc/core_t/analyzer/interface/keymgr_interface.h" #include "ypc/core_t/analyzer/internal/data_streams/multi_data_stream.h" #include "ypc/core_t/analyzer/internal/data_streams/noinput_data_stream.h" #include "ypc/core_t/analyzer/internal/data_streams/raw_data_stream.h" #include "ypc/core_t/analyzer/internal/data_streams/sealed_data_stream.h" +#include "ypc/core_t/analyzer/internal/data_streams/oram_sealed_data_stream.h" +#include "ypc/core_t/analyzer/internal/data_streams/convert_sealed_data_stream.h" #include "ypc/core_t/analyzer/internal/keymgr_session.h" #include "ypc/core_t/analyzer/raw_data_provider.h" #include "ypc/core_t/analyzer/sealed_data_provider.h" +#include "ypc/core_t/analyzer/oram_sealed_data_provider.h" #include "ypc/core_t/analyzer/var/data_source_var.h" #include "ypc/corecommon/nt_cols.h" #include "ypc/corecommon/package.h" #include "ypc/stbox/ebyte.h" #include "ypc/stbox/stx_status.h" +#include "ypc/common/crypto_prefix.h" namespace ypc { namespace internal { @@ -89,6 +93,126 @@ class data_interface } }; +template +class data_interface + : virtual public data_source_var, + virtual public keymgr_interface, + virtual public keymgr_session { + typedef keymgr_interface keymgr_interface_t; + typedef Crypto ecc; +public: + uint32_t init_data_source(const uint8_t *data_source_info, uint32_t len) { + using ntt = nt; + auto pkg = make_package::type>::from_bytes(data_source_info, len); + auto ret = keymgr_session::init_keymgr_session_oram(); + if (ret) { + LOG(ERROR) << "init_keymgr_session_oram failed: " << stbox::status_string(ret); + return ret; + } + stbox::bytes private_key, dian_pkey; + ret = keymgr_interface_t::request_private_key_for_public_key( + pkg.get(), private_key, dian_pkey); + if (ret) { + LOG(ERROR) << "request_private_key_for_public_key failed: " + << stbox::status_string(ret); + return ret; + } + + // decrypt param + stbox::bytes param_private_key, d_pkey; + ret = keymgr_interface_t::request_private_key_for_public_key( + pkg.get(), param_private_key, d_pkey); + if (ret) { + LOG(ERROR) << "request_private_key_for_public_key failed: " + << stbox::status_string(ret); + return ret; + } + + stbox::bytes decrypted_param; + ret = ecc::decrypt_message_with_prefix(param_private_key, + pkg.get(), decrypted_param, + ypc::utc::crypto_prefix_arbitrary); + if (ret) { + LOG(ERROR) << "decrypt_message_with_prefix failed: " + << stbox::status_string(ret); + return ret; + } + + m_ds_use_pkey = pkg.get() + pkg.get(); + m_datasource.reset(new oram_sealed_data_provider( + pkg.get(), private_key, pkg.get(), decrypted_param)); + return stbox::stx_status::success; + } + + uint32_t check_actual_data_hash() { + auto p = m_datasource.get(); + if(p->data_hash().size() != p->expect_data_hash().size()) { + LOG(ERROR) << "the expected data hash array needs to contain " + << p->expect_data_hash().size() << " data hashes, but got " + << p->data_hash().size(); + return stbox::stx_status::data_hash_not_same_as_expect; + } + + for(uint32_t i = 0; i < p->data_hash().size(); ++i) { + if(p->data_hash()[i] != p->expect_data_hash()[i]) { + LOG(ERROR) << "expect " << i <<"-th data hash is " << p->expect_data_hash()[i] << ", got " + << p->data_hash()[i]; + return stbox::stx_status::data_hash_not_same_as_expect; + } + } + + return stbox::stx_status::success; + } +}; + +template +class data_interface + : virtual public data_source_var, + virtual public keymgr_interface, + virtual public keymgr_session { + typedef keymgr_interface keymgr_interface_t; + +public: + uint32_t init_data_source(const uint8_t *data_source_info, uint32_t len) { + using ntt = nt; + auto pkg = make_package::type>::from_bytes(data_source_info, len); + + auto ret = keymgr_session::init_keymgr_session(); + if (ret) { + LOG(ERROR) << "init_keymgr_session_convert failed: " << stbox::status_string(ret); + return ret; + } + + stbox::bytes private_key, dian_pkey; + ret = keymgr_interface_t::request_private_key_for_public_key( + pkg.get(), private_key, dian_pkey); + if (ret) { + LOG(ERROR) << "request_private_key_for_public_key failed: " + << stbox::status_string(ret); + return ret; + } + + m_ds_use_pkey = pkg.get() + pkg.get(); + + m_datasource.reset(new sealed_data_provider( + pkg.get(), private_key)); + + return stbox::stx_status::success; + } + + uint32_t check_actual_data_hash() { + auto p = m_datasource.get(); + if (p->data_hash() != p->expect_data_hash()) { + LOG(ERROR) << "expect " << p->expect_data_hash() << ", got " + << p->data_hash(); + return stbox::stx_status::data_hash_not_same_as_expect; + } + return stbox::stx_status::success; + } +}; + template class data_interface : virtual public data_source_var, @@ -102,7 +226,7 @@ class data_interface auto pkg = make_package::type>::from_bytes(data_source_info, len); - auto ret = keymgr_session::init_keymgr_session(); + auto ret = keymgr_session::init_keymgr_session_oram(); if (ret) { LOG(ERROR) << "init_keymgr_session failed: " << stbox::status_string(ret); return ret; diff --git a/include/ypc/core_t/analyzer/internal/data_streams/convert_sealed_data_stream.h b/include/ypc/core_t/analyzer/internal/data_streams/convert_sealed_data_stream.h new file mode 100644 index 00000000..a31cfd4c --- /dev/null +++ b/include/ypc/core_t/analyzer/internal/data_streams/convert_sealed_data_stream.h @@ -0,0 +1,16 @@ +#pragma once +#include "ypc/core_t/analyzer/helper/parser_type_traits.h" +#include "ypc/stbox/ebyte.h" + +namespace ypc { +namespace internal { + +class convert_sealed_data_stream {}; + +} // namespace internal +using convert_sealed_data_stream = internal::convert_sealed_data_stream; + +template <> struct datasource_type_traits { + constexpr static uint32_t value = ypc::utc::single_sealed_datasource_parser; +}; +} // namespace ypc \ No newline at end of file diff --git a/include/ypc/core_t/analyzer/internal/data_streams/oram_sealed_data_stream.h b/include/ypc/core_t/analyzer/internal/data_streams/oram_sealed_data_stream.h new file mode 100644 index 00000000..781cfb92 --- /dev/null +++ b/include/ypc/core_t/analyzer/internal/data_streams/oram_sealed_data_stream.h @@ -0,0 +1,16 @@ +#pragma once +#include "ypc/core_t/analyzer/helper/parser_type_traits.h" +#include "ypc/stbox/ebyte.h" + +namespace ypc { +namespace internal { + +class oram_sealed_data_stream {}; + +} // namespace internal +using oram_sealed_data_stream = internal::oram_sealed_data_stream; + +template <> struct datasource_type_traits { + constexpr static uint32_t value = ypc::utc::oram_sealed_datasource_parser; +}; +} // namespace ypc \ No newline at end of file diff --git a/include/ypc/core_t/analyzer/internal/keymgr_session.h b/include/ypc/core_t/analyzer/internal/keymgr_session.h index f12a309a..500a9bec 100644 --- a/include/ypc/core_t/analyzer/internal/keymgr_session.h +++ b/include/ypc/core_t/analyzer/internal/keymgr_session.h @@ -18,6 +18,7 @@ class keymgr_session : virtual public enclave_hash_var, virtual public keymgr_var { protected: uint32_t init_keymgr_session(); + uint32_t init_keymgr_session_oram(); uint32_t close_keymgr_session(); }; } // namespace internal diff --git a/include/ypc/core_t/analyzer/oasm_lib.h b/include/ypc/core_t/analyzer/oasm_lib.h new file mode 100644 index 00000000..75375f35 --- /dev/null +++ b/include/ypc/core_t/analyzer/oasm_lib.h @@ -0,0 +1,10 @@ +#pragma once +#include + +extern "C" { + void omove(uint32_t i, uint32_t *item, uint32_t loc, uint32_t *leaf, uint32_t newLabel); + void oset_value(uint32_t *dest, uint32_t value, bool flag); + void oset_flag(bool *dest_flag, bool src_flag, bool flag); + void oset_bytes(uint8_t *dest_bytes, uint8_t *src_bytes, uint32_t bytes_size, bool flag); +} + diff --git a/include/ypc/core_t/analyzer/oram_sealed_data_provider.h b/include/ypc/core_t/analyzer/oram_sealed_data_provider.h new file mode 100644 index 00000000..bb8020b1 --- /dev/null +++ b/include/ypc/core_t/analyzer/oram_sealed_data_provider.h @@ -0,0 +1,881 @@ +#pragma once +#include "ypc/core_t/analyzer/data_source.h" +#include "ypc/corecommon/oram_types.h" +#include +#include "ypc/common/crypto_prefix.h" +#include "ypc/core_t/analyzer/oasm_lib.h" + +#include "hpda/extractor/extractor_base.h" +#include "ypc/common/limits.h" +#include "ypc/core_t/analyzer/eparser_t_interface.h" +#include "ypc/core_t/ecommon/package.h" +#include "ypc/corecommon/package.h" +#include "ypc/stbox/ebyte.h" +#include "ypc/stbox/stx_common.h" +#include "ypc/stbox/tsgx/channel/dh_session_initiator.h" +#include "ypc/stbox/tsgx/log.h" +#include + + +using oram_ntt = ypc::oram::nt; +typedef ypc::nt ntt; + +namespace ypc { +template +class oram_sealed_data_provider : public data_source_with_merkle_hash { + typedef Crypto crypto; +public: + oram_sealed_data_provider(const stbox::bytes &data_hash, + const stbox::bytes &private_key, + const stbox::bytes &public_key, + const stbox::bytes &decrypted_param) + : data_source_with_merkle_hash(data_hash), m_private_key(private_key), + m_public_key(public_key), m_decrypted_param(decrypted_param) { + // magic string here, Do Not Change! + m_header.stash_size = oram::stash_size; + + } + + virtual ~oram_sealed_data_provider() {} + + + virtual bool process() { + if(! m_is_access_executed) { + bool ret = access(); + if (!ret) { + LOG(ERROR) << "Failed to get target batch"; + } + + m_is_access_executed = true; + return true; + } + if(m_item_index + 1 < m_valid_item_num) { + m_item_index++; + return true; + } + + return false; + } + + virtual data_source_output_t output_value() { + data_source_output_t ret; + ret.set::data>(m_items[m_item_index]); + return ret; + } + + virtual const std::vector &data_hash() const { return m_actual_data_hash; } + const bytes &private_key() const { return m_private_key; } + +private: + + bool access() { + bool ret = download_oram_params(); + if (!ret) { + LOG(ERROR) << "Failed to download_oram_params\n"; + return false; + } + + uint32_t block_id; + ret = get_block_id(block_id); + if (!ret) { + LOG(ERROR) << "Failed to get block_id\n"; + return false; + } + + ret = download_position_map(); + if (!ret) { + LOG(ERROR) << "Failed to download position map\n"; + return false; + } + + unsigned char random_value[4]; + sgx_status_t se_ret = sgx_read_rand((unsigned char*) random_value, 4); + if (se_ret != SGX_SUCCESS) { + LOG(ERROR) << "Failed to generate rand number\n"; + return false; + } + uint32_t new_leaf = *((uint32_t *)random_value) % (1 << m_header.level_num_L) + 1; + uint32_t leaf; + + oarray_search(m_position_map, block_id, &leaf, new_leaf, m_position_map.size()); + + ret = download_merkle_hash(leaf); + if (!ret) { + LOG(ERROR) << "Failed to download merkle hash\n"; + return false; + } + + ret = download_path(leaf); + if (!ret) { + LOG(ERROR) << "Failed to download path\n"; + return false; + } + + ret = download_stash(); + if (!ret) { + LOG(ERROR) << "Failed to download stash\n"; + return false; + } + + ret = decrypt_path(); + if (!ret) { + LOG(ERROR) << "Failed to decrypt path\n"; + return false; + } + + ret = access_in_stash(block_id, new_leaf); + if (!ret) { + LOG(ERROR) << "Failed to access in stash\n"; + return false; + } + + rebuild_new_path(leaf); + + ret = recalculate_hash(); + if (!ret) { + LOG(ERROR) << "Failed to recalculate hash\n"; + return false; + } + + ret = encrypt_path(); + if (!ret) { + LOG(ERROR) << "Failed to encrypt path\n"; + return false; + } + + ret = update_position_map(); + if (!ret) { + LOG(ERROR) << "Failed to update position map\n"; + return false; + } + + ret = upload_path(leaf); + if (!ret) { + LOG(ERROR) << "Failed to upload path\n"; + return false; + } + + ret = update_merkle_hash(leaf); + if (!ret) { + LOG(ERROR) << "Failed to update merkle hash\n"; + return false; + } + + ret = update_stash(); + if (!ret) { + LOG(ERROR) << "Failed to update stash\n"; + return false; + } + + return true; + } + + // oarray_search(m_position_map, block_id, &leaf, new_leaf, m_position_map.size()); + void oarray_search(std::vector& array, uint32_t loc, uint32_t *leaf, + uint32_t newLabel, uint32_t N_level) { + for(uint32_t i=0; i < N_level; ++i) { + omove(i, &(array[i]), loc, leaf, newLabel); + } + return; + } + + bool download_oram_params() { + auto ret = stbox::ocall_cast(download_oram_params_OCALL) + (m_expect_root_hash.data(), m_expect_root_hash.size(), &m_header.block_num, + &m_header.bucket_num_N, &m_header.level_num_L, + &m_header.bucket_str_size, &m_header.batch_str_size); + if (ret != stbox::stx_status::success) { + return false; + } + + for(uint32_t i = 0; i < m_header.stash_size; ++i) { + oram_ntt::block_t s_block; + s_block.set + (0, 0, 0, stbox::bytes(m_header.batch_str_size)); + m_stash.push_back(s_block); + } + + return true; + } + + bool get_block_id(uint32_t &block_id) { + uint32_t b_id; + bytes param_hash; + crypto::hash_256(m_decrypted_param, param_hash); + + auto ret = stbox::ocall_cast(get_block_id_OCALL) + (m_expect_root_hash.data(), m_expect_root_hash.size(), + &b_id, param_hash.data(), param_hash.size()); + if (ret != stbox::stx_status::success) { + return false; + } + + block_id = b_id; + return true; + } + + bool download_position_map() { + uint8_t *posmap; + uint32_t posmap_len; + + auto ret = stbox::ocall_cast(download_position_map_OCALL) + (m_expect_root_hash.data(), m_expect_root_hash.size(), &posmap, &posmap_len); + if (ret != stbox::stx_status::success) { + return false; + } + + stbox::bytes posmap_str(posmap_len); + memcpy(posmap_str.data(), posmap, posmap_len); + + stbox::bytes decrypted_position_map_bytes; + uint32_t status = crypto::decrypt_message_with_prefix( + m_private_key, posmap_str, decrypted_position_map_bytes, ypc::utc::crypto_prefix_arbitrary); + if (ret) { + LOG(ERROR) << "decrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + try { + oram_ntt::position_map_t position_map_pkg = make_package::from_bytes(decrypted_position_map_bytes); + m_position_map = position_map_pkg.get(); + } catch (const std::exception &e) { + LOG(ERROR) << "make_package got: " << e.what(); + return false; + } + + return true; + } + + bool update_position_map() { + oram_ntt::position_map_t position_map_pkg; + position_map_pkg.set(m_position_map); + stbox::bytes position_map_bytes; + try { + position_map_bytes = make_bytes::for_package(position_map_pkg); + } catch (const std::exception &e) { + LOG(ERROR) << "make_bytes got: " << e.what(); + return false; + } + stbox::bytes encrypted_position_map_bytes; + uint32_t status = crypto::encrypt_message_with_prefix( + m_public_key, position_map_bytes, ypc::utc::crypto_prefix_arbitrary, encrypted_position_map_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + auto ret = stbox::ocall_cast(update_position_map_OCALL) + (m_expect_root_hash.data(), m_expect_root_hash.size(), + encrypted_position_map_bytes.data(), encrypted_position_map_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "update_position_map_OCALL fail!"; + return false; + } + + return true; + } + + bool download_merkle_hash(uint32_t leaf) { + uint8_t *merkle_hash; + uint32_t merkle_hash_len; + + auto ret = stbox::ocall_cast(download_merkle_hash_OCALL) + (m_expect_root_hash.data(), m_expect_root_hash.size(), + leaf, &merkle_hash, &merkle_hash_len); + if (ret != stbox::stx_status::success) { + return false; + } + + bytes merkle_hash_str(merkle_hash_len); + memcpy(merkle_hash_str.data(), merkle_hash, merkle_hash_len); + + try { + oram_ntt::merkle_hash_pkg_t merkle_hash_pkg = + make_package::from_bytes(merkle_hash_str); + m_merkle_hash_array = merkle_hash_pkg.get(); + } catch (const std::exception &e) { + LOG(ERROR) << "make_package got: " << e.what(); + return false; + } + + for(const auto& hash_p : m_merkle_hash_array) { + if(hash_p.get()) { + m_expect_data_hash.push_back(hash_p.get()); + } + } + + return true; + } + + bool download_path(uint32_t leaf) { + uint8_t *encrypted_path; + uint32_t encrypted_path_len; + + auto ret = stbox::ocall_cast(download_path_OCALL) + (m_expect_root_hash.data(), m_expect_root_hash.size(), + leaf, &encrypted_path, &encrypted_path_len); + if (ret != stbox::stx_status::success) { + return false; + } + + m_encrypted_path = stbox::bytes(encrypted_path_len); + memcpy(m_encrypted_path.data(), encrypted_path, encrypted_path_len); + + return true; + } + + bool download_stash() { + uint8_t *stash; + uint32_t stash_len; + + auto ret = stbox::ocall_cast(download_stash_OCALL) + (m_expect_root_hash.data(), m_expect_root_hash.size(), + &stash, &stash_len); + if (ret != stbox::stx_status::success) { + return false; + } + + if(stash_len > 0) { + stbox::bytes stash_str(stash_len); + memcpy(stash_str.data(), stash, stash_len); + + stbox::bytes decrypted_stash_bytes; + uint32_t status = crypto::decrypt_message_with_prefix( + m_private_key, stash_str, decrypted_stash_bytes, ypc::utc::crypto_prefix_arbitrary); + if (status) { + LOG(ERROR) << "decrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + try { + auto stash_pkg = make_package::from_bytes(decrypted_stash_bytes); + auto block_array = stash_pkg.get(); + + uint32_t i = 0; + for(const auto& element : block_array) { + m_stash[i++] = element; + } + + } catch (const std::exception &e) { + LOG(ERROR) << "make_package got: " << e.what(); + return false; + } + + } + + return true; + } + + bool decrypt_path() { + std::vector bucket_array; + + try { + oram_ntt::path_pkg_t path_pkg = make_package::from_bytes(m_encrypted_path); + bucket_array = path_pkg.get(); + } catch (const std::exception &e) { + LOG(ERROR) << "make_package got: " << e.what(); + return false; + } + + // calculate m_actual_data_hash + for(const auto& encrypted_bucket_str : bucket_array) { + stbox::bytes decrypted_bucket_str; + uint32_t status = crypto::decrypt_message_with_prefix( + m_private_key, encrypted_bucket_str, decrypted_bucket_str, ypc::utc::crypto_prefix_arbitrary); + if (status) { + LOG(ERROR) << "decrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + stbox::bytes data_hash; + crypto::hash_256(bytes("Fidelius"), data_hash); + + std::vector block_array; + try { + oram_ntt::bucket_pkg_t bucket_pkg = make_package::from_bytes(decrypted_bucket_str); + m_decrypted_path.push_back(bucket_pkg); + block_array = bucket_pkg.get(); + } catch (const std::exception &e) { + LOG(ERROR) << "make_package got: " << e.what(); + return false; + } + + for(oram_ntt::block_t e_block : block_array) { + bool pushed_flag = false; + for(uint32_t k = 0; k < m_stash.size(); ++k) { + uint32_t p_block_id = e_block.get(); + uint32_t s_block_id = m_stash[k].get(); + + bool flag = p_block_id > 0 && s_block_id == 0 && !pushed_flag; + + uint32_t p_leaf_label = e_block.get(); + uint32_t s_leaf_label = m_stash[k].get(); + + uint32_t p_valid_item_num = e_block.get(); + uint32_t s_valid_item_num = m_stash[k].get(); + + stbox::bytes p_encrypted_batch = e_block.get(); + stbox::bytes s_encrypted_batch = m_stash[k].get(); + + oset_value(&s_block_id, p_block_id, flag); + oset_value(&s_leaf_label, p_leaf_label, flag); + oset_value(&s_valid_item_num, p_valid_item_num, flag); + oset_bytes(s_encrypted_batch.data(), p_encrypted_batch.data(), p_encrypted_batch.size(), flag); + oset_flag(&pushed_flag, true, flag); + + m_stash[k].set + (s_block_id, s_leaf_label, s_valid_item_num, s_encrypted_batch); + + + + // if(e_block.get() > 0 && m_stash[k].get() == 0) { + // m_stash[k].set + // (e_block.get(), e_block.get(), + // e_block.get(), e_block.get()); + // break; + // } + } + + stbox::bytes encrypted_batch = e_block.get(); + stbox::bytes decrypted_batch_str; + status = crypto::decrypt_message_with_prefix( + m_private_key, encrypted_batch, decrypted_batch_str, ypc::utc::crypto_prefix_arbitrary); + if (status) { + LOG(ERROR) << "decrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + try { + auto pkg = make_package::from_bytes(decrypted_batch_str); + auto items = pkg.get(); + + for (auto b : items) { + stbox::bytes k_hash = data_hash + b; + crypto::hash_256(k_hash, data_hash); + } + + } catch (const std::exception &e) { + LOG(ERROR) << "make_package got: " << e.what(); + return false; + } + } + + m_actual_data_hash.push_back(data_hash); + + + } + + for(int i = m_actual_data_hash.size() - 2; i >= 0; --i) { + stbox::bytes k_hash = m_actual_data_hash[i] + m_merkle_hash_array[2*i + 1].get(); + crypto::hash_256(k_hash, m_actual_data_hash[i]); + + k_hash = m_actual_data_hash[i] + m_merkle_hash_array[2*i + 2].get(); + crypto::hash_256(k_hash, m_actual_data_hash[i]); + } + + return true; + } + + void oset_value(uint32_t *dest_value, uint32_t src_value, bool move_flag) { + if(move_flag) { + *dest_value = src_value; + } + } + + bool access_in_stash(uint32_t block_id, uint32_t new_leaf) { + stbox::bytes target_encrypted_batch_str(m_header.batch_str_size); + for(uint32_t i = 0; i < m_stash.size(); ++i) { + bool flag = m_stash[i].get() == block_id; + uint32_t leaf_label = m_stash[i].get(); + + oset_value(&leaf_label, new_leaf, flag); + oset_value(&m_valid_item_num, m_stash[i].get(), flag); + oset_bytes(target_encrypted_batch_str.data(), m_stash[i].get().data(), m_header.batch_str_size, flag); + m_stash[i].set(leaf_label); + + } + + if(m_valid_item_num == 0) { + LOG(ERROR) << "fail, the value of valid item num is 0 "; + return false; + } + + stbox::bytes decrypted_batch_str; + uint32_t status = crypto::decrypt_message_with_prefix( + m_private_key, target_encrypted_batch_str, decrypted_batch_str, ypc::utc::crypto_prefix_arbitrary); + if (status) { + LOG(ERROR) << "decrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + try { + auto pkg = make_package::from_bytes(decrypted_batch_str); + m_items = pkg.get(); + if (m_items.size() == 0) { + LOG(ERROR) << "fail, m_items size should not be 0 "; + return false; + } + + m_item_index = 0; + return true; + } catch (const std::exception &e) { + LOG(ERROR) << "make_package got: " << e.what(); + return false; + } + + + // for(uint32_t i = 0; i < m_stash.size(); ++i) { + // if(m_stash[i].get() == block_id) { + // m_valid_item_num = m_stash[i].get(); + // if(m_valid_item_num == 0) { + // LOG(ERROR) << "fail, the value of valid item num is 0 "; + // return false; + // } + + // stbox::bytes encrypted_batch_str = m_stash[i].get(); + // stbox::bytes decrypted_batch_str; + // uint32_t status = crypto::decrypt_message_with_prefix( + // m_private_key, encrypted_batch_str, decrypted_batch_str, ypc::utc::crypto_prefix_arbitrary); + // if (status) { + // LOG(ERROR) << "decrypt_message_with_prefix fail: " + // << stbox::status_string(status); + // return false; + // } + + // try { + // auto pkg = make_package::from_bytes(decrypted_batch_str); + // m_items = pkg.get(); + // if (m_items.size() == 0) { + // LOG(ERROR) << "fail, m_items size should not be 0 "; + // return false; + // } + + // m_item_index = 0; + // m_stash[i].set(new_leaf); + // return true; + // } catch (const std::exception &e) { + // LOG(ERROR) << "make_package got: " << e.what(); + // return false; + // } + + // } + + // } + + return false; + } + + // uint8_t get_level(uint32_t leaf1, uint32_t leaf2) { + // uint32_t leaf1_index = leaf1 - 1 + (1 << m_header.level_num_L) - 1; + // uint32_t leaf2_index = leaf2 - 1 + (1 << m_header.level_num_L) - 1; + + // while(leaf1_index != leaf2_index) { + // leaf1_index = (leaf1_index - 1) / 2; + // leaf2_index = (leaf2_index - 1) / 2; + // } + + // return floor(log2(leaf1_index + 1)); + // } + + bool fit_in_path(uint32_t bucket_index, uint32_t leaf1, uint32_t leaf2) { + uint32_t leaf1_index = leaf1 - 1 + (1 << m_header.level_num_L) - 1; + uint32_t leaf2_index = leaf2 - 1 + (1 << m_header.level_num_L) - 1; + + while(leaf1_index != leaf2_index) { + leaf1_index = (leaf1_index - 1) / 2; + leaf2_index = (leaf2_index - 1) / 2; + } + + return leaf1_index == bucket_index; + } + + void rebuild_new_path(uint32_t leaf) { + uint32_t bucket_index = leaf - 1 + (1 << m_header.level_num_L) - 1; + for(int i = m_decrypted_path.size() - 1; i >= 0; --i) { + auto block_array = m_decrypted_path[i].get(); + for(uint8_t j = 0; j < oram::BucketSizeZ; ++j) { + uint32_t b_block_id = 0, b_leaf_label = 0, b_valid_item_num = 0; + stbox::bytes encrypted_batch = block_array[j].get(); + bool updated_flag = false; + for(uint32_t k = 0; k < m_stash.size(); ++k) { + uint32_t s_block_id = m_stash[k].get(); + uint32_t s_leaf_label = m_stash[k].get(); + uint32_t s_valid_item_num = m_stash[k].get(); + bool flag = s_block_id > 0 && !updated_flag + && fit_in_path(bucket_index, s_leaf_label, leaf); + + oset_value(&b_block_id, s_block_id, flag); + oset_value(&b_leaf_label, s_leaf_label, flag); + oset_value(&b_valid_item_num, s_valid_item_num, flag); + oset_bytes(encrypted_batch.data(), m_stash[k].get().data(), + encrypted_batch.size(), flag);\ + + oset_value(&s_block_id, 0, flag); + oset_value(&s_leaf_label, 0, flag); + oset_value(&s_valid_item_num, 0, flag); + + oset_flag(&updated_flag, true, flag); + + m_stash[k].set + (s_block_id, s_leaf_label, s_valid_item_num); + } + + block_array[j].set + (b_block_id, b_leaf_label, b_valid_item_num, encrypted_batch); + + } + m_decrypted_path[i].set(block_array); + bucket_index = (bucket_index - 1) / 2; + } + + + // for(auto &bu : m_decrypted_path) { + // auto block_array = bu.get(); + // for(uint8_t j = 0; j < oram::BucketSizeZ; ++j) { + // block_array[j].set(0, 0, 0); + // } + // bu.set(block_array); + // } + + // for(uint32_t i = 0; i < m_stash.size(); ++i) { + // if(m_stash[i].get() > 0) { + // uint8_t low_level = get_level(leaf, m_stash[i].get()); + // for(int level = low_level; level >= 0; --level) { + // for(uint8_t k = 0; k < oram::BucketSizeZ; ++k) { + // if(m_decrypted_path[level].get()[k].get() == 0) { + // m_decrypted_path[level].get()[k].set + // + // (m_stash[i].get(), m_stash[i].get(), + // m_stash[i].get(), m_stash[i].get()); + + // m_stash[i].set(0, 0, 0); + + // break; + + // } + // } + + // if(m_stash[i].get() == 0) { + // break; + // } + // } + // } + // } + } + + bool update_stash() { + std::vector stash_block_array; + for(uint32_t i = 0; i < m_stash.size(); ++i) { + if (m_stash[i].get() > 0) { + stash_block_array.push_back(m_stash[i]); + } + } + + + oram_ntt::bucket_pkg_t stash_pkg; + stash_pkg.set(stash_block_array); + stbox::bytes stash_str; + try { + stash_str = make_bytes::for_package(stash_pkg); + } catch (const std::exception &e) { + LOG(ERROR) << "make_bytes got: " << e.what(); + return false; + } + stbox::bytes encrypted_stash_str; + uint32_t status = crypto::encrypt_message_with_prefix( + m_public_key, stash_str, ypc::utc::crypto_prefix_arbitrary, encrypted_stash_str); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + auto ret = stbox::ocall_cast(update_stash_OCALL) + (m_expect_root_hash.data(), m_expect_root_hash.size(), + encrypted_stash_str.data(), encrypted_stash_str.size()); + + if (ret != stbox::stx_status::success) { + return false; + } + + return true; + } + + bool recalculate_hash() { + uint32_t index = 0; + for(auto &bu : m_decrypted_path) { + + stbox::bytes data_hash; + crypto::hash_256(bytes("Fidelius"), data_hash); + + auto block_array = bu.get(); + for(auto &e_block : block_array) { + stbox::bytes encrypted_batch = e_block.get(); + stbox::bytes decrypted_batch_str; + uint32_t status = crypto::decrypt_message_with_prefix( + m_private_key, encrypted_batch, decrypted_batch_str, ypc::utc::crypto_prefix_arbitrary); + if (status) { + LOG(ERROR) << "decrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + try { + auto pkg = make_package::from_bytes(decrypted_batch_str); + auto items = pkg.get(); + + for (auto b : items) { + stbox::bytes k_hash = data_hash + b; + crypto::hash_256(k_hash, data_hash); + } + } catch (const std::exception &e) { + LOG(ERROR) << "make_package got: " << e.what(); + return false; + } + + } + + while(!m_merkle_hash_array[index].get()) { + ++index; + } + m_merkle_hash_array[index++].set(data_hash); + + } + + for(int i = m_header.level_num_L - 1; i >= 0; --i) { + + if(i != 0 && m_merkle_hash_array[2*i - 1].get()) { + stbox::bytes k_hash = m_merkle_hash_array[2*i - 1].get() + + m_merkle_hash_array[2*i + 1].get(); + stbox::bytes data_hash; + crypto::hash_256(k_hash, data_hash); + + k_hash = data_hash + m_merkle_hash_array[2*i + 2].get(); + crypto::hash_256(k_hash, data_hash); + + m_merkle_hash_array[2*i - 1].set(data_hash); + } + + if(m_merkle_hash_array[2*i].get()) { + stbox::bytes k_hash = m_merkle_hash_array[2*i].get() + + m_merkle_hash_array[2*i + 1].get(); + stbox::bytes data_hash; + crypto::hash_256(k_hash, data_hash); + + k_hash = data_hash + m_merkle_hash_array[2*i + 2].get(); + crypto::hash_256(k_hash, data_hash); + + m_merkle_hash_array[2*i].set(data_hash); + } + } + + return true; + } + + bool encrypt_path() { + std::vector encrypted_path_array; + for(oram_ntt::bucket_pkg_t bucket_pkg : m_decrypted_path) { + stbox::bytes bucket_str; + try { + bucket_str = make_bytes::for_package(bucket_pkg); + } catch (const std::exception &e) { + LOG(ERROR) << "make_bytes got: " << e.what(); + return false; + } + stbox::bytes encrypted_bucket_bytes; + uint32_t status = crypto::encrypt_message_with_prefix( + m_public_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + encrypted_path_array.push_back(encrypted_bucket_bytes); + } + + oram_ntt::path_pkg_t path_pkg; + path_pkg.set(encrypted_path_array); + try { + m_encrypted_path = make_bytes::for_package(path_pkg); + } catch (const std::exception &e) { + LOG(ERROR) << "make_bytes got: " << e.what(); + return false; + } + + return true; + } + + bool upload_path(uint32_t leaf) { + auto ret = stbox::ocall_cast(upload_path_OCALL) + (m_expect_root_hash.data(), m_expect_root_hash.size(), + leaf, m_encrypted_path.data(), m_encrypted_path.size()); + if (ret != stbox::stx_status::success) { + return false; + } + + return true; + } + + bool update_merkle_hash(uint32_t leaf) { + oram_ntt::merkle_hash_pkg_t merkle_hash_pkg; + merkle_hash_pkg.set(m_merkle_hash_array); + bytes merkle_hash_str; + try { + merkle_hash_str = make_bytes::for_package(merkle_hash_pkg); + } catch (const std::exception &e) { + LOG(ERROR) << "make_bytes got: " << e.what(); + return false; + } + + auto ret = stbox::ocall_cast(update_merkle_hash_OCALL) + (m_expect_root_hash.data(), m_expect_root_hash.size(), + leaf, merkle_hash_str.data(), merkle_hash_str.size()); + if (ret != stbox::stx_status::success) { + return false; + } + + return true; + } + +protected: + struct oram_header { + uint32_t stash_size; + uint32_t block_num; + uint32_t bucket_num_N; + uint8_t level_num_L; + uint32_t bucket_str_size; + uint32_t row_length; + uint32_t batch_str_size; + }; + + std::vector m_actual_data_hash; + std::vector m_items; + size_t m_item_index; + bytes m_private_key; + bytes m_public_key; + bytes m_decrypted_param; + + oram_header m_header; + std::vector m_stash; + std::vector m_position_map; + stbox::bytes m_encrypted_path; + std::vector m_decrypted_path; + uint32_t m_valid_item_num; + std::vector m_merkle_hash_array; + bool m_is_access_executed; + + +}; +} // namespace ypc \ No newline at end of file diff --git a/include/ypc/core_t/analyzer/var/data_hash_var.h b/include/ypc/core_t/analyzer/var/data_hash_var.h index 7b3047b5..dbbc83e0 100644 --- a/include/ypc/core_t/analyzer/var/data_hash_var.h +++ b/include/ypc/core_t/analyzer/var/data_hash_var.h @@ -7,5 +7,11 @@ class data_hash_var { protected: stbox::bytes m_data_hash; }; + +class data_merkle_hash_var { +protected: + std::vector m_data_hash; +}; + } // namespace internal } // namespace ypc diff --git a/include/ypc/core_t/analyzer/var/data_source_var.h b/include/ypc/core_t/analyzer/var/data_source_var.h index 65065b53..6ffe3270 100644 --- a/include/ypc/core_t/analyzer/var/data_source_var.h +++ b/include/ypc/core_t/analyzer/var/data_source_var.h @@ -1,5 +1,6 @@ #pragma once #include "ypc/core_t/analyzer/data_source.h" +#include "ypc/core_t/analyzer/internal/data_streams/oram_sealed_data_stream.h" #include "ypc/core_t/analyzer/internal/data_streams/multi_data_stream.h" #include "ypc/core_t/analyzer/internal/data_streams/noinput_data_stream.h" #include "ypc/stbox/ebyte.h" @@ -13,6 +14,12 @@ template class data_source_var { stbox::bytes m_ds_use_pkey; }; +template <> class data_source_var { +protected: + std::shared_ptr m_datasource; + stbox::bytes m_ds_use_pkey; +}; + template <> class data_source_var { protected: std::vector> m_datasource; diff --git a/include/ypc/core_t/analyzer/yaenclave_t_interface.h b/include/ypc/core_t/analyzer/yaenclave_t_interface.h index 47e2bfe3..eb3cc12d 100644 --- a/include/ypc/core_t/analyzer/yaenclave_t_interface.h +++ b/include/ypc/core_t/analyzer/yaenclave_t_interface.h @@ -32,6 +32,22 @@ sgx_status_t SGX_CDECL km_send_request_ocall( secure_message_t *resp_message, size_t resp_message_size); sgx_status_t SGX_CDECL km_end_session_ocall(uint32_t *retval, uint32_t session_id); + +sgx_status_t SGX_CDECL km_session_request_oram_ocall(uint32_t *retval, + sgx_dh_msg1_t *dh_msg1, + uint32_t *session_id); +sgx_status_t SGX_CDECL km_exchange_report_oram_ocall(uint32_t *retval, + sgx_dh_msg2_t *dh_msg2, + sgx_dh_msg3_t *dh_msg3, + uint32_t session_id); +sgx_status_t SGX_CDECL km_send_request_oram_ocall( + uint32_t *retval, uint32_t session_id, secure_message_t *req_message, + size_t req_message_size, size_t max_payload_size, + secure_message_t *resp_message, size_t resp_message_size); +sgx_status_t SGX_CDECL km_end_session_oram_ocall(uint32_t *retval, + uint32_t session_id); + + sgx_status_t SGX_CDECL sgx_oc_cpuidex(int cpuinfo[4], int leaf, int subleaf); sgx_status_t SGX_CDECL sgx_thread_wait_untrusted_event_ocall(int *retval, const void *self); diff --git a/include/ypc/corecommon/nt_cols.h b/include/ypc/corecommon/nt_cols.h index b139ade1..56d56055 100644 --- a/include/ypc/corecommon/nt_cols.h +++ b/include/ypc/corecommon/nt_cols.h @@ -46,6 +46,12 @@ template struct nt { using multi_sealed_data_info_t = ::ff::util::ntobject; + define_nt(param_data_pkey, BytesType); + using oram_sealed_data_info_t = ::ff::util::ntobject; + define_nt(oram_sealed_data_info_vector, std::vector); + using multi_oram_sealed_data_info_t = + ::ff::util::ntobject; + using batch_data_pkg_t = ::ff::net::ntpackage<0x82c4e8d8, batch_data>; define_nt(encrypted_param, BytesType); diff --git a/include/ypc/corecommon/oram_types.h b/include/ypc/corecommon/oram_types.h new file mode 100644 index 00000000..538c145e --- /dev/null +++ b/include/ypc/corecommon/oram_types.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include + + +namespace ypc { +namespace oram { + +constexpr static uint8_t BucketSizeZ = 4; +constexpr static uint32_t stash_size = 90; + +struct header { + uint32_t block_num; + uint32_t bucket_num_N; + uint8_t level_num_L; + uint32_t bucket_str_size; + uint32_t batch_str_size; + long int id_map_filepos; + long int oram_tree_filepos; + long int position_map_filepos; + long int merkle_tree_filepos; + long int stash_filepos; + uint64_t stash_size; + + uint64_t item_num_each_batch; + uint64_t item_size; + +}; + +template struct nt { + define_nt(item_index_field_hash, BytesType); + define_nt(block_id, uint32_t); + using id_map_pair = ::ff::util::ntobject; + define_nt(id_map, std::vector); + using id_map_t = ::ff::net::ntpackage<0x82c4e8da, id_map>; + + define_nt(position_map, std::vector); + using position_map_t = ::ff::net::ntpackage<0x82c4e8dd, position_map>; + + define_nt(leaf_label, uint32_t); + define_nt(valid_item_num, uint32_t); + define_nt(encrypted_batch, BytesType); + using block_t = ::ff::util::ntobject; + + define_nt(bucket, std::vector); + using bucket_pkg_t = ::ff::net::ntpackage<0x82c4e8df, bucket>; + + define_nt(path, std::vector); + using path_pkg_t = ::ff::net::ntpackage<0x82c4e8ea, path>; + + define_nt(data_hash, BytesType); + define_nt(in_path, bool); + using hash_pair = ::ff::util::ntobject; + define_nt(merkle_hash, std::vector); + using merkle_hash_pkg_t = ::ff::net::ntpackage<0x82c4e7ea, merkle_hash>; +}; + + + + + +} +} \ No newline at end of file diff --git a/include/ypc/edl/eparser.edl b/include/ypc/edl/eparser.edl index e3481d32..fd6bec3b 100644 --- a/include/ypc/edl/eparser.edl +++ b/include/ypc/edl/eparser.edl @@ -39,7 +39,33 @@ enclave { /* define OCALLs here. */ uint32_t next_data_batch([in, size=hash_size] const uint8_t * data_hash, uint32_t hash_size, [out] uint8_t ** data, [out] uint32_t *len); - + void free_data_batch([user_check] uint8_t *data); + + uint32_t download_oram_params_OCALL([in, size=hash_size] const uint8_t *data_hash, uint32_t hash_size, + [out] uint32_t *block_num, [out] uint32_t *bucket_num_N, + [out]uint8_t *level_num_L, [out] uint32_t *bucket_str_size, + [out] uint32_t *batch_str_size); + uint32_t get_block_id_OCALL([in, size=hash_size] const uint8_t * data_hash, uint32_t hash_size, + [out] uint32_t *block_id, + [in, size=hash_size] const uint8_t *param_hash, uint32_t param_hash_size); + uint32_t download_position_map_OCALL([in, size=hash_size] const uint8_t *data_hash, uint32_t hash_size, + [out] uint8_t ** position_map, [out] uint32_t *len); + uint32_t update_position_map_OCALL([in, size=hash_size] const uint8_t *data_hash, uint32_t hash_size, + [in, size = len] uint8_t * position_map, uint32_t len); + uint32_t download_path_OCALL([in, size=hash_size] const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, [out] uint8_t ** encrpypted_path, [out] uint32_t *len); + uint32_t download_stash_OCALL([in, size=hash_size] const uint8_t *data_hash, uint32_t hash_size, + [out] uint8_t ** stash, [out] uint32_t *len); + uint32_t update_stash_OCALL([in, size=hash_size] const uint8_t *data_hash, uint32_t hash_size, + [in, size = len] uint8_t * stash, uint32_t len); + uint32_t upload_path_OCALL([in, size=hash_size] const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, [in, size = len] uint8_t * encrpypted_path, uint32_t len); + uint32_t download_merkle_hash_OCALL([in, size=hash_size] const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, [out] uint8_t ** merkle_hash, [out] uint32_t *len); + uint32_t update_merkle_hash_OCALL([in, size=hash_size] const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, [in, size = len] uint8_t * merkle_hash, uint32_t len); + uint32_t write_convert_data_structure(int64_t filepos, [in, size = len] uint8_t * convert_data_bytes, uint32_t len); + uint32_t download_convert_params_ocall([out] uint32_t *block_num, [out] long int *oram_tree_filepos, [out] uint64_t *item_num_each_batch, [out] uint64_t *item_size); }; }; diff --git a/include/ypc/edl/yaenclave.edl b/include/ypc/edl/yaenclave.edl index 8e81c15e..c81949eb 100644 --- a/include/ypc/edl/yaenclave.edl +++ b/include/ypc/edl/yaenclave.edl @@ -16,5 +16,10 @@ enclave { uint32_t km_exchange_report_ocall([in] sgx_dh_msg2_t *dh_msg2, [out] sgx_dh_msg3_t *dh_msg3, uint32_t session_id); uint32_t km_send_request_ocall(uint32_t session_id, [in, size = req_message_size] secure_message_t* req_message, size_t req_message_size, size_t max_payload_size, [out, size=resp_message_size] secure_message_t* resp_message, size_t resp_message_size); uint32_t km_end_session_ocall(uint32_t session_id); + + uint32_t km_session_request_oram_ocall([out] sgx_dh_msg1_t *dh_msg1,[out] uint32_t *session_id); + uint32_t km_exchange_report_oram_ocall([in] sgx_dh_msg2_t *dh_msg2, [out] sgx_dh_msg3_t *dh_msg3, uint32_t session_id); + uint32_t km_send_request_oram_ocall(uint32_t session_id, [in, size = req_message_size] secure_message_t* req_message, size_t req_message_size, size_t max_payload_size, [out, size=resp_message_size] secure_message_t* resp_message, size_t resp_message_size); + uint32_t km_end_session_oram_ocall(uint32_t session_id); }; }; diff --git a/include/ypc/stbox/stx_status.def b/include/ypc/stbox/stx_status.def index 0b8b8ccf..54d36f99 100644 --- a/include/ypc/stbox/stx_status.def +++ b/include/ypc/stbox/stx_status.def @@ -44,6 +44,8 @@ ATT_STATUS(data_hash_not_same_as_expect, 0xC5) ATT_STATUS(aes_rand_fail, 0xC6) ATT_STATUS(forward_target_not_set, 0xC7) ATT_STATUS(forward_allowance_invalid, 0xC8) +ATT_STATUS(oram_sealed_file_error, 0xC9) +ATT_STATUS(convert_parser_error, 0xCA) ATT_STATUS(allowance_use_inconsistent_pkey, 0xD0) ATT_STATUS(model_allowance_miss, 0xD1) ATT_STATUS(data_allowance_miss, 0xD2) diff --git a/include/ypc/version.h b/include/ypc/version.h index a89cba33..e1d79f42 100644 --- a/include/ypc/version.h +++ b/include/ypc/version.h @@ -1,11 +1,11 @@ #pragma once #include "ypc/common/version.h" -#define YPC_CORE_T_VERSION ypc::version(0, 5, 1) -#define YPC_RUNTIME_VERSION ypc::version(0, 5, 1) -#define YPC_STBOX_VERSION ypc::version(0, 5, 1) -#define YPC_ECC_T_VERSION ypc::version(0, 5, 1) -#define YPC_KEYMGR_T_VERSION ypc::version(0, 5, 1) +#define YPC_CORE_T_VERSION ypc::version(0, 5, 2) +#define YPC_RUNTIME_VERSION ypc::version(0, 5, 2) +#define YPC_STBOX_VERSION ypc::version(0, 5, 2) +#define YPC_ECC_T_VERSION ypc::version(0, 5, 2) +#define YPC_KEYMGR_T_VERSION ypc::version(0, 5, 2) -#define YPC_KEYMGR_RUNTIME_VERSION ypc::version(0, 5, 1) +#define YPC_KEYMGR_RUNTIME_VERSION ypc::version(0, 5, 2) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 865107e8..dac1fdb4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,11 +3,7 @@ include_directories(${PROJECT_SOURCE_DIR}/test/) include_directories(${FF_INCLUDE_DIR}) link_directories(${FF_LIB_DIR}) -file(GLOB sub_folders ./*) -foreach(folder ${sub_folders}) - if(IS_DIRECTORY ${folder}) - if(EXISTS ${folder}/CMakeLists.txt) - add_subdirectory(${folder}) - endif() - endif() -endforeach() +add_subdirectory(core) +add_subdirectory(crypto) +add_subdirectory(keymgr) +add_subdirectory(toolkit) diff --git a/test/integrate/classic_job_convert.py b/test/integrate/classic_job_convert.py new file mode 100644 index 00000000..42a8497d --- /dev/null +++ b/test/integrate/classic_job_convert.py @@ -0,0 +1,223 @@ +#!/usr/bin/python3 +import common +import json +from job_step import job_step + +def create_oram_file(oram_sealed_data_url): + cmd = "touch {}".format(oram_sealed_data_url) + output = common.execute_cmd(cmd) + return [cmd, output] + +class classic_job_convert: + def __init__(self, crypto, name, data_url, convert_parser_url, convert_parser2_url, oram_parser_url, plugin_url, con_read_num, config={}): + self.crypto = crypto + self.name = name + self.data_url = data_url + self.convert_parser_url = convert_parser_url + self.convert_parser2_url = convert_parser2_url + self.oram_parser_url = oram_parser_url + self.plugin_url = plugin_url + # self.input = input_param + self.all_outputs = list() + self.config = config + + self.input = "" + self.data_shukey_json = {} + self.shukey_json = {} + self.con_read_num = con_read_num + self.read_count = 0 + self.data_hash = "" + self.enclave_hash = "" + + + # def convert(self): + + + def run(self): + ''' + 1. call terminus to generate key + 2. call data provider to seal data + 3. call terminus to generate forward message + 4. call terminus to generate request + 5. call fid_analyzer + 6. call terminus to decrypt + ''' + self.all_outputs = list() + + # 0. generate key + data_key_file = self.name + ".data.key.json" + if self.read_count == 0: + self.data_shukey_json = job_step.gen_key(self.crypto, data_key_file) + if self.read_count + 1 == self.con_read_num: + self.all_outputs.append(data_key_file) + + + # 1. generate key + key_file = self.name + ".key.json" + if self.read_count == 0: + self.shukey_json = job_step.gen_key(self.crypto, key_file) + if self.read_count + 1 == self.con_read_num: + self.all_outputs.append(key_file) + + # 2. call data provider to seal data + sealed_data_url = self.name + ".sealed" + sealed_output = self.name + ".sealed.output" + oram_sealed_data_url = sealed_data_url + ".oram" + summary = {} + summary['data-url'] = self.data_url + summary['plugin-path'] = self.plugin_url + summary['sealed-data-url'] = sealed_data_url + summary['sealed-output'] = sealed_output + + if self.read_count == 0: + r = job_step.seal_data(self.crypto, self.data_url, self.plugin_url, + sealed_data_url, sealed_output, data_key_file) + self.data_hash = job_step.read_data_hash(sealed_output) + summary['data-hash'] = self.data_hash + print("done seal data with hash: {}, cmd: {}".format(self.data_hash, r[0])) + create_oram_file(oram_sealed_data_url) + + if self.read_count + 1 == self.con_read_num: + self.all_outputs.append(sealed_data_url) + self.all_outputs.append(sealed_output) + + # use first pkey + key = job_step.get_first_key(self.crypto) + pkey = key['public-key'] + summary['tee-pkey'] = key['public-key'] + + # 3. call terminus to generate forward message + forward_result = self.name + ".shukey.foward.json" + data_forward_json = job_step.forward_message( + self.crypto, data_key_file, pkey, "", forward_result) + if self.read_count == 0: + self.enclave_hash = job_step.read_parser_hash(self.convert_parser_url) + if self.read_count == 1: + self.enclave_hash = job_step.read_parser_hash(self.convert_parser2_url) + if self.read_count > 1: + self.enclave_hash = job_step.read_parser_hash(self.oram_parser_url) + self.all_outputs.append(forward_result) + + # 4.0 call terminus to generate forward message + param_key_forward_result = self.name + ".request.shukey.foward.json" + rq_forward_json = job_step.forward_message( + self.crypto, key_file, pkey, self.enclave_hash, param_key_forward_result) + self.all_outputs.append(param_key_forward_result) + + # 4.1 call terminus to generate request + param_output_url = self.name + "_param.json" + if self.read_count <= 1: + param_json = job_step.generate_request( + self.crypto, self.input, data_key_file, param_output_url, self.config) + else: + param_json = job_step.generate_request( + self.crypto, self.input, key_file, param_output_url, self.config) + summary['analyzer-input'] = param_json["encrypted-input"] + self.all_outputs.append(param_output_url) + + # 5. call fid_convert_analyzer 转换文件格式 + if self.read_count == 0: + input_obj = { + "input_data_url": sealed_data_url, + "input_data_hash": self.data_hash, + "shu_info": { + "shu_pkey": self.data_shukey_json["public-key"], + "encrypted_shu_skey": data_forward_json["encrypted_skey"], + "shu_forward_signature": data_forward_json["forward_sig"], + "enclave_hash": data_forward_json["enclave_hash"] + }, + "public-key": self.data_shukey_json["public-key"], + "tag": "0" + } + input_data = [input_obj] + parser_input_file = self.name + "parser_input.json" + parser_output_file = self.name + "parser_output.json" + result_json = job_step.fid_convert_analyzer(self.shukey_json, self.data_shukey_json, rq_forward_json, self.enclave_hash, input_data, + self.convert_parser_url, pkey, {}, self.crypto, param_json, [], parser_input_file, parser_output_file) + + summary['encrypted-result'] = result_json["encrypted_result"] + summary["result-signature"] = result_json["result_signature"] + summary["cost-signature"] = result_json["cost_signature"] + with open(self.name + ".summary.json", "w") as of: + json.dump(summary, of) + self.all_outputs.append(parser_input_file) + self.all_outputs.append(parser_output_file) + + + if self.read_count == 1: + input_obj = { + "input_data_url": sealed_data_url, + "input_data_hash": self.data_hash, + "shu_info": { + "shu_pkey": self.data_shukey_json["public-key"], + "encrypted_shu_skey": data_forward_json["encrypted_skey"], + "shu_forward_signature": data_forward_json["forward_sig"], + "enclave_hash": data_forward_json["enclave_hash"] + }, + "public-key": self.data_shukey_json["public-key"], + "tag": "0" + } + input_data = [input_obj] + parser_input_file = self.name + "parser_input.json" + parser_output_file = self.name + "parser_output.json" + result_json = job_step.fid_convert_analyzer(self.shukey_json, self.data_shukey_json, rq_forward_json, self.enclave_hash, input_data, + self.convert_parser2_url, pkey, {}, self.crypto, param_json, [], parser_input_file, parser_output_file) + + summary['encrypted-result'] = result_json["encrypted_result"] + summary["result-signature"] = result_json["result_signature"] + summary["cost-signature"] = result_json["cost_signature"] + with open(self.name + ".summary.json", "w") as of: + json.dump(summary, of) + self.all_outputs.append(parser_input_file) + self.all_outputs.append(parser_output_file) + + + # 6. call fid_oram_analyzer 使用ORAM格式的加密文件进行分析任务 + if self.read_count > 1: + root_hash_url = self.name + ".root_hash" + r = job_step.get_root_hash(oram_sealed_data_url, root_hash_url) + root_hash = r[1] + + input_obj = { + "input_data_url": oram_sealed_data_url, + # "input_data_hash": data_hash, + "input_data_hash": root_hash, + "shu_info": { + "shu_pkey": self.data_shukey_json["public-key"], + "encrypted_shu_skey": data_forward_json["encrypted_skey"], + "shu_forward_signature": data_forward_json["forward_sig"], + "enclave_hash": data_forward_json["enclave_hash"] + }, + "public-key": self.data_shukey_json["public-key"], + "tag": "0" + } + input_data = [input_obj] + parser_input_file = self.name + "parser_input.json" + parser_output_file = self.name + "parser_output.json" + result_json = {} + # if(self.name.split('_')[-1] == 'oram'): + result_json = job_step.fid_oram_analyzer(self.shukey_json, rq_forward_json, self.enclave_hash, input_data, + self.oram_parser_url, pkey, {}, self.crypto, param_json, [], parser_input_file, parser_output_file) + + summary['encrypted-result'] = result_json["encrypted_result"] + summary["result-signature"] = result_json["result_signature"] + summary["cost-signature"] = result_json["cost_signature"] + with open(self.name + ".summary.json", "w") as of: + json.dump(summary, of) + self.all_outputs.append(parser_input_file) + self.all_outputs.append(parser_output_file) + self.all_outputs.append(root_hash_url) + + # 6. call terminus to decrypt + encrypted_result = summary["encrypted-result"] + decrypted_result = self.name + ".result" + self.result = job_step.decrypt_result( + self.crypto, encrypted_result, key_file, decrypted_result) + self.all_outputs.append(decrypted_result) + + + if 'remove-files' in self.config and self.config['remove-files']: + job_step.remove_files(self.all_outputs) + + self.read_count += 1 + diff --git a/test/integrate/classic_job_oram.py b/test/integrate/classic_job_oram.py new file mode 100644 index 00000000..41a77ff3 --- /dev/null +++ b/test/integrate/classic_job_oram.py @@ -0,0 +1,139 @@ +#!/usr/bin/python3 +import common +import os +import json +from job_step import job_step + + +class classic_job_oram: + def __init__(self, crypto, name, data_url, parser_url, plugin_url, con_read_num, config={}): + self.crypto = crypto + self.name = name + self.data_url = data_url + self.parser_url = parser_url + self.plugin_url = plugin_url + self.input = "" + self.all_outputs = list() + self.config = config + self.data_shukey_json = {} + self.shukey_json = {} + self.con_read_num = con_read_num + self.read_count = 0 + + def run(self): + ''' + 1. call terminus to generate key + 2. call data provider to seal data + 3. call terminus to generate forward message + 4. call terminus to generate request + 5. call fid_analyzer + 6. call terminus to decrypt + ''' + self.all_outputs = list() + + # 0. generate key + data_key_file = self.name + ".data.key.json" + if self.read_count == 0: + self.data_shukey_json = job_step.gen_key(self.crypto, data_key_file) + if self.read_count + 1 == self.con_read_num: + self.all_outputs.append(data_key_file) + + # 1. generate key + key_file = self.name + ".key.json" + if self.read_count == 0: + self.shukey_json = job_step.gen_key(self.crypto, key_file) + if self.read_count + 1 == self.con_read_num: + self.all_outputs.append(key_file) + + # 2. call data provider to seal data + sealed_data_url = self.name + ".sealed.oram" + sealed_output = self.name + ".sealed.oram.output" + summary = {} + summary['data-url'] = self.data_url + summary['plugin-path'] = self.plugin_url + summary['sealed-data-url'] = sealed_data_url + summary['sealed-output'] = sealed_output + + if self.read_count == 0: + r = job_step.oram_seal_data(self.crypto, self.data_url, self.plugin_url, + sealed_data_url, sealed_output, data_key_file) + # 需要在sealed_data_url中获取root hash ，作为fid_oram_analyzer的参数之一 + root_hash_url = self.name + ".root_hash" + r = job_step.get_root_hash(sealed_data_url, root_hash_url) + root_hash = r[1] + + + # root_hash = job_step.read_root_hash(root_hash_url) + # data_hash = job_step.read_data_hash(sealed_output) + # summary['data-hash'] = data_hash + if self.read_count + 1 == self.con_read_num: + self.all_outputs.append(sealed_data_url) + self.all_outputs.append(sealed_output) + self.all_outputs.append(root_hash_url) + + + # use first pkey + key = job_step.get_first_key(self.crypto) + pkey = key['public-key'] + summary['tee-pkey'] = key['public-key'] + + # 3. call terminus to generate forward message + forward_result = self.name + ".shukey.foward.json" + data_forward_json = job_step.forward_message( + self.crypto, data_key_file, pkey, "", forward_result) + enclave_hash = job_step.read_parser_hash(self.parser_url) + self.all_outputs.append(forward_result) + + # 4.0 call terminus to generate forward message + param_key_forward_result = self.name + ".request.shukey.foward.json" + rq_forward_json = job_step.forward_message( + self.crypto, key_file, pkey, enclave_hash, param_key_forward_result) + self.all_outputs.append(param_key_forward_result) + + # 4.1 call terminus to generate request + param_output_url = self.name + "_param.json" + param_json = job_step.generate_request( + self.crypto, self.input, key_file, param_output_url, self.config) + summary['analyzer-input'] = param_json["encrypted-input"] + self.all_outputs.append(param_output_url) + + # 5. call fid_oram_analyzer + input_obj = { + "input_data_url": sealed_data_url, + # "input_data_hash": data_hash, + "input_data_hash": root_hash, + "shu_info": { + "shu_pkey": self.data_shukey_json["public-key"], + "encrypted_shu_skey": data_forward_json["encrypted_skey"], + "shu_forward_signature": data_forward_json["forward_sig"], + "enclave_hash": data_forward_json["enclave_hash"] + }, + "public-key": self.data_shukey_json["public-key"], + "tag": "0" + } + input_data = [input_obj] + parser_input_file = self.name + "parser_input.json" + parser_output_file = self.name + "parser_output.json" + result_json = {} + # if(self.name.split('_')[-1] == 'oram'): + result_json = job_step.fid_oram_analyzer(self.shukey_json, rq_forward_json, enclave_hash, input_data, + self.parser_url, pkey, {}, self.crypto, param_json, [], parser_input_file, parser_output_file) + + summary['encrypted-result'] = result_json["encrypted_result"] + summary["result-signature"] = result_json["result_signature"] + summary["cost-signature"] = result_json["cost_signature"] + with open(self.name + ".summary.json", "w") as of: + json.dump(summary, of) + self.all_outputs.append(parser_input_file) + self.all_outputs.append(parser_output_file) + + # 6. call terminus to decrypt + encrypted_result = summary["encrypted-result"] + decrypted_result = self.name + ".result" + self.result = job_step.decrypt_result( + self.crypto, encrypted_result, key_file, decrypted_result) + self.all_outputs.append(decrypted_result) + if 'remove-files' in self.config and self.config['remove-files']: + job_step.remove_files(self.all_outputs) + + self.read_count += 1 diff --git a/test/integrate/common.py b/test/integrate/common.py index 9f877fbc..b95f5082 100644 --- a/test/integrate/common.py +++ b/test/integrate/common.py @@ -94,6 +94,19 @@ def fid_data_provider(**kwargs): output = execute_cmd(cmd) return [cmd, output] +def fid_oram_data_provider(**kwargs): + cmd = os.path.join(bin_dir, "./oram_data_provider") + for k, v in kwargs.items(): + cmd = cmd + " --{} {}".format(k, v) + output = execute_cmd(cmd) + return [cmd, output] + +def fid_get_root_hash(**kwargs): + cmd = os.path.join(bin_dir, "./get_root_hash") + for k, v in kwargs.items(): + cmd = cmd + " --{} {}".format(k, v) + output = execute_cmd(cmd) + return [cmd, output] def fid_dump(**kwargs): cmd = os.path.join(bin_dir, "./ydump") @@ -119,6 +132,13 @@ def fid_analyzer(**kwargs): output = execute_cmd(cmd) return [cmd, output] +def fid_oram_analyzer(**kwargs): + cmd = os.path.join(bin_dir, "./fid_oram_analyzer") + cmd = "GLOG_logtostderr=1 " + cmd + for k, v in kwargs.items(): + cmd = cmd + " --{} {}".format(k, v) + output = execute_cmd(cmd) + return [cmd, output] def iris_data(**kwargs): cmd = os.path.join(bin_dir, "./iris_gen_classify_input") diff --git a/test/integrate/generate_convert.py b/test/integrate/generate_convert.py new file mode 100644 index 00000000..4163cdde --- /dev/null +++ b/test/integrate/generate_convert.py @@ -0,0 +1,789 @@ +import os +import common +import subprocess + +def get_crypto_name(crypto): + if(crypto == "stdeth"): + crypto_name = "ypc::crypto::eth_sgx_crypto" + if(crypto == "gmssl"): + crypto_name = "ypc::crypto::gmssl_sgx_crypto" + return crypto_name + + +def compile_in_subdir(build_dir): + # build_dir + makefile_path = os.path.join(build_dir, 'Makefile') + if not os.path.isfile(makefile_path): + print(f"Error: makefile not found in {build_dir}") + return False + + # 执行编译命令 + try: + result = subprocess.run(['make', '-j8'], cwd = build_dir, check = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + print(result.stdout.decode('utf-8')) + return True + except subprocess.CalledProcessError as e: + print(f"Error: {e.stderr.decode('utf-8')}") + return False + + +def recompile(mode): + cmd = os.path.join(common.sdk_dir, "./build.sh compile-project {} -j8").format(mode) + output = common.execute_cmd(cmd) + return [cmd, output] + + +def run_test_convert_parser(): + cmd = 'python3 {}/test/integrate/test_convert_parser.py'.format(common.sdk_dir) + output = common.execute_cmd(cmd) + return [cmd, output] + + +# 检查目录是否存在 +def check_directory(directory_path): + if os.path.exists(directory_path): + return True + else: + print(f"\"{directory_path}\" does not exist.") + return False + +# 进入目录 +def enter_directory(directory_path): + if check_directory(directory_path): + os.chdir(directory_path) + return True + else: + return False + + +def gen_convert_parser(file_path, crypto_name, index_name): + content = '''#include "ypc/corecommon/package.h" +#include "ypc/stbox/ebyte.h" +#include "ypc/stbox/stx_common.h" +#ifdef EXAMPLE_FM_NORMAL +#include +typedef ypc::bytes bytes; +#else +#include "ypc/core_t/analyzer/data_source.h" +#include "ypc/stbox/tsgx/log.h" +typedef stbox::bytes bytes; +#endif +#include "user_type.h" +#include "ypc/corecommon/data_source.h" +#include "ypc/corecommon/to_type.h" +#include +#include +#include +#include + + +#include "ypc/core_t/analyzer/eparser_t_interface.h" +#include "ypc/common/crypto_prefix.h" +#include "ypc/corecommon/crypto/gmssl.h" +#include "ypc/corecommon/crypto/stdeth.h" +#include "ypc/corecommon/oram_types.h" +#include +#include + + +using oram_ntt = ypc::oram::nt; +using ntt = ypc::nt; + + + + + +define_nt(input_buf, std::string); +typedef ff::net::ntpackage<0, input_buf> input_buf_t; + + +class crypto_base { +public: + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) = 0; + virtual uint32_t hash_256(const bytes &msg, bytes &hash) = 0; +}; +using crypto_ptr_t = std::shared_ptr; +template class crypto_tool : public crypto_base { +public: + using crypto_t = Crypto; + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) { + return crypto_t::encrypt_message_with_prefix(public_key, data, prefix, + cipher); + } + virtual uint32_t hash_256(const bytes &msg, bytes &hash) { + return crypto_t::hash_256(msg, hash); + } +}; + + +class convert_parser { +public: + convert_parser() {} + convert_parser(ypc::data_source *source) : m_source(source){}; + + inline bytes do_parse(const bytes ¶m) { + bytes result; + + LOG(INFO) << "do convert_parse"; + ypc::to_type converter(m_source); + + crypto_ptr_t crypto_ptr = std::make_shared>(); + bytes pub_key(param); + + + + LOG(INFO) << "create id_map"; + + size_t batch_size = 0; + size_t item_size = 0; + uint64_t batch_num = 0; // the number of batch + uint64_t full_num = 0; + uint64_t last_num = 0; + uint64_t item_num = 0; + + std::vector id_map_array; + uint32_t batch_id = 1; + + hpda::processor::internal::filter_impl match2( + &converter, [&](const user_item_t &v) { + if(item_size == 0) { + typename ypc::cast_obj_to_package::type pt = v; + auto item_data = ypc::make_bytes::for_package(pt); + item_size = item_data.size(); + } + + std::string item_index_field = v.get<''' + + content = content + index_name + + content = content + '''>(); + + input_buf_t item_index_field_pkg; + item_index_field_pkg.set(item_index_field); + bytes item_index_field_bytes = ypc::make_bytes::for_package(item_index_field_pkg); + bytes item_index_field_hash; + crypto_ptr->hash_256(item_index_field_bytes, item_index_field_hash); + + std::shared_ptr k_v(new oram_ntt::id_map_pair()); + k_v->set(item_index_field_hash, batch_id); + id_map_array.push_back(*k_v); + + ++item_num; + + batch_size += item_size; + if (batch_size >= ypc::utc::max_item_size) { + + if(full_num == 0) { + full_num = item_num; + } + + item_num = 0; + batch_size = 0; + + ++batch_id; + ++batch_num; + } + + return false; + }); + match2.get_engine()->run(); + + if(item_num > 0) { + last_num = item_num; + ++batch_num; + } + + oram_ntt::id_map_t id_map_pkg; + id_map_pkg.set(id_map_array); + id_map_array.clear(); + bytes id_map_bytes = ypc::make_bytes::for_package(id_map_pkg); + + + LOG(INFO) << "write header"; + + ypc::oram::header osf_header{}; + osf_header.block_num = batch_num; + uint32_t real_bucket_num = ceil(static_cast(osf_header.block_num) / ypc::oram::BucketSizeZ); + osf_header.level_num_L = ceil(log2(real_bucket_num + 1)) - 1; + osf_header.bucket_num_N = (1 << (osf_header.level_num_L + 1)) - 1; + osf_header.id_map_filepos = sizeof(osf_header); + osf_header.oram_tree_filepos = osf_header.id_map_filepos + id_map_bytes.size(); + + osf_header.item_size = item_size; + if(full_num > 0) { + osf_header.item_num_each_batch = full_num; + } else { + osf_header.item_num_each_batch = last_num; + } + + auto ret = stbox::ocall_cast(write_convert_data_structure) + (0, (uint8_t *)&osf_header, sizeof(osf_header)); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + + LOG(INFO) << "write id map"; + + int32_t id_map_size = 8000000; + for(int i = 0; i <= id_map_bytes.size(); i += id_map_size) { + if(i + id_map_size <= id_map_bytes.size()) { + ret = stbox::ocall_cast(write_convert_data_structure) + (osf_header.id_map_filepos + i, id_map_bytes.data() + i, id_map_size); + } else { + ret = stbox::ocall_cast(write_convert_data_structure) + (osf_header.id_map_filepos + i, id_map_bytes.data() + i, id_map_bytes.size() - i); + } + + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + } + + + return result; + } + +protected: + ypc::data_source *m_source; +}; +''' + + with open(file_path, "w") as file: + file.write(content) + + + +def gen_convert_parser2(file_path2, crypto_name): + content = '''#include "ypc/corecommon/package.h" +#include "ypc/stbox/ebyte.h" +#include "ypc/stbox/stx_common.h" +#ifdef EXAMPLE_FM_NORMAL +#include +typedef ypc::bytes bytes; +#else +#include "ypc/core_t/analyzer/data_source.h" +#include "ypc/stbox/tsgx/log.h" +typedef stbox::bytes bytes; +#endif +#include "user_type.h" +#include "ypc/corecommon/data_source.h" +#include "ypc/corecommon/to_type.h" +#include +#include +#include +#include + + +#include "ypc/core_t/analyzer/eparser_t_interface.h" +#include "ypc/common/crypto_prefix.h" +#include "ypc/corecommon/crypto/gmssl.h" +#include "ypc/corecommon/crypto/stdeth.h" +#include "ypc/corecommon/oram_types.h" +#include +#include + + +using oram_ntt = ypc::oram::nt; +using ntt = ypc::nt; + + + + + +define_nt(input_buf, std::string); +typedef ff::net::ntpackage<0, input_buf> input_buf_t; + + +class crypto_base { +public: + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) = 0; + virtual uint32_t hash_256(const bytes &msg, bytes &hash) = 0; +}; +using crypto_ptr_t = std::shared_ptr; +template class crypto_tool : public crypto_base { +public: + using crypto_t = Crypto; + virtual uint32_t encrypt_message_with_prefix(const bytes &public_key, + const bytes &data, + uint32_t prefix, + bytes &cipher) { + return crypto_t::encrypt_message_with_prefix(public_key, data, prefix, + cipher); + } + virtual uint32_t hash_256(const bytes &msg, bytes &hash) { + return crypto_t::hash_256(msg, hash); + } +}; + + +bytes random_string(size_t len) { + std::string ret(len, '0'); + static std::default_random_engine generator; + static std::uniform_int_distribution distribution(int('a'), int('z')); + static auto rand = std::bind(distribution, generator); + + for (size_t i = 0; i < len; i++) { + ret[i] = rand(); + } + return bytes(ret.data(), ret.size()); +} + +bool push_dummy_block(std::vector& bucket_array, bytes &data_hash, + uint8_t count, uint64_t item_num_each_batch, uint64_t item_size, + const crypto_ptr_t &crypto_ptr, const bytes &public_key) { + for(uint8_t i = 0; i < count; ++i) { + oram_ntt::block_t b_block; + + std::vector dummy_batch; + for(uint32_t j = 0; j < item_num_each_batch; ++j) { + bytes dummy_item = random_string(item_size); + dummy_batch.push_back(dummy_item); + bytes k_hash = data_hash + dummy_item; + crypto_ptr->hash_256(k_hash, data_hash); + } + + bytes encrypted_dummy_batch; + bytes dummy_batch_str = + ypc::make_bytes::for_package(dummy_batch); + + // encrypt dummy batch + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, dummy_batch_str, ypc::utc::crypto_prefix_arbitrary, encrypted_dummy_batch); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + b_block.set(0, 0, 0, encrypted_dummy_batch); + bucket_array.push_back(b_block); + } + return true; +} + +uint32_t get_leaf_label(uint32_t bucket_index, uint8_t level_num_L) { + // leftmost leaf node + uint32_t leftmost_leaf_index = (1 << level_num_L) - 1; + if(bucket_index >= leftmost_leaf_index) { + return bucket_index - leftmost_leaf_index + 1; + } + + // randomly select a path to the leaf node + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(0, 1); + + if(dis(gen) == 0) { + return get_leaf_label(2 * bucket_index + 1, level_num_L); + } + return get_leaf_label(2 * bucket_index + 2, level_num_L); +} + +bool push_real_block(std::vector& bucket_array, bytes &data_hash, + uint32_t& block_id_value, uint32_t bucket_index, + std::vector &position_map_array, uint8_t level_num_L, + std::vector &batch, uint32_t &batch_str_size, + uint64_t item_num_each_batch, uint64_t item_size, + const crypto_ptr_t &crypto_ptr, const bytes &public_key) { + oram_ntt::block_t b_block; + uint32_t valid_item_num = batch.size(); + for(uint32_t i = 0; i < item_num_each_batch - valid_item_num; ++i) { + ypc::bytes item = random_string(item_size); + batch.push_back(item); + } + + for(auto &item : batch) { + ypc::bytes k_hash = data_hash + item; + crypto_ptr->hash_256(k_hash, data_hash); + } + + bytes encrypted_batch; + ypc::bytes batch_str = + ypc::make_bytes::for_package(batch); + // encrypt batch + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, batch_str, ypc::utc::crypto_prefix_arbitrary, encrypted_batch); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return false; + } + + if(batch_str_size != encrypted_batch.size()) { + batch_str_size = encrypted_batch.size(); + } + + uint32_t b_leaf_label = get_leaf_label(bucket_index, level_num_L); + position_map_array[block_id_value] = b_leaf_label; + b_block.set + (block_id_value++, b_leaf_label, valid_item_num, encrypted_batch); + bucket_array.push_back(b_block); + + return true; +} + + +class convert_parser2 { +public: + convert_parser2() {} + convert_parser2(ypc::data_source *source) : m_source(source){}; + + inline bytes do_parse(const bytes ¶m) { + bytes result; + LOG(INFO) << "do convert_parse2"; + ypc::to_type converter(m_source); + + crypto_ptr_t crypto_ptr = std::make_shared>(); + bytes pub_key(param); + + + // 1. read header + ypc::oram::header osf_header{}; + + auto ret = stbox::ocall_cast(download_convert_params_ocall) + (&osf_header.block_num, &osf_header.oram_tree_filepos, &osf_header.item_num_each_batch, &osf_header.item_size); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "download_convert_params_ocall fail!"; + return result; + } + + uint32_t real_bucket_num = ceil(static_cast(osf_header.block_num) / ypc::oram::BucketSizeZ); + osf_header.level_num_L = ceil(log2(real_bucket_num + 1)) - 1; + osf_header.bucket_num_N = (1 << (osf_header.level_num_L + 1)) - 1; + osf_header.id_map_filepos = sizeof(osf_header); + + + // 2. write ORAM tree + LOG(INFO) << "write ORAM tree"; + std::vector data_hash_array; + + // from which bucket to start writing real blocks + uint8_t lastbucket_realblocknum = osf_header.block_num % ypc::oram::BucketSizeZ; + uint32_t bucket_index = 0; // bucket index in ORAM tree + uint32_t block_id_value = 1; // block_id_value <= osf_header.block_num + + + // 2.1 write buckets full of dummy blocks + LOG(INFO) << "write buckets full of dummy blocks"; + // osf.seekp(osf_header.oram_tree_filepos, osf.beg); + int64_t filepos = osf_header.oram_tree_filepos; + for(uint32_t i = 0; i < osf_header.bucket_num_N - real_bucket_num; ++i) { + std::vector bucket_array; + bytes data_hash; + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + bool retf = push_dummy_block(bucket_array, data_hash, ypc::oram::BucketSizeZ, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + if(!retf) { + LOG(ERROR) << "push_dummy_block fail!"; + return result; + } + + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = ypc::make_bytes::for_package(bucket_pkg); + + // secondary encryption on the serialized bucket + // in order to encrypt the mapping relationship between block_id and leaf_label + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return result; + } + + ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + + filepos += encrypted_bucket_bytes.size(); + data_hash_array.push_back(data_hash); + ++bucket_index; + } + + + + std::vector batch; + std::vector position_map_array(osf_header.block_num + 1, 0); + + + LOG(INFO) << "write real blocks"; + // 2.2 write the bucket that contains both real and dummy blocks + std::vector bucket_array; + bytes data_hash; + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + int i = ypc::oram::BucketSizeZ; + + if(lastbucket_realblocknum != 0) { + --real_bucket_num; + push_dummy_block(bucket_array, data_hash, + ypc::oram::BucketSizeZ - lastbucket_realblocknum, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + i = lastbucket_realblocknum; + } + + bool break_flag = false; + int j = 0; + batch.clear(); + + + // 2.3 write buckets full of real blocks + hpda::processor::internal::filter_impl match3( + &converter, [&](const user_item_t &v) { + typename ypc::cast_obj_to_package::type pt = v; + auto item_data = ypc::make_bytes::for_package(pt); + + if(break_flag) { + return false; + } + + batch.push_back(item_data); + ++j; + + if(j == osf_header.item_num_each_batch) { + + bool retf = push_real_block(bucket_array, data_hash, block_id_value, bucket_index, + position_map_array, osf_header.level_num_L, batch, osf_header.batch_str_size, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + if(!retf) { + LOG(ERROR) << "push_dummy_block fail!"; + break_flag = true; + return false; + } + + --i; + + if(i == 0) { + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = ypc::make_bytes::for_package(bucket_pkg); + + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + break_flag = true; + return false; + } + + if(osf_header.bucket_str_size != encrypted_bucket_bytes.size()) { + osf_header.bucket_str_size = encrypted_bucket_bytes.size(); + } + + auto ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + break_flag = true; + return false; + } + + + filepos += encrypted_bucket_bytes.size(); + data_hash_array.push_back(data_hash); + ++bucket_index; + bucket_array.clear(); + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + + i = ypc::oram::BucketSizeZ; + } + + batch.clear(); + j = 0; + } + + return false; + }); + match3.get_engine()->run(); + + if(break_flag) { + LOG(ERROR) << "write real blocks fail!"; + return result; + } + + // 2.4 write the last data block + // The number of valid rows in the last data block may be less than item_num_each_batch + if(j > 0) { + bool retf = push_real_block(bucket_array, data_hash, block_id_value, bucket_index, + position_map_array, osf_header.level_num_L, batch, osf_header.batch_str_size, + osf_header.item_num_each_batch, osf_header.item_size, crypto_ptr, pub_key); + if(!retf) { + LOG(ERROR) << "push_dummy_block fail!"; + break_flag = true; + return result; + } + + --i; + + if(i == 0) { + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = ypc::make_bytes::for_package(bucket_pkg); + + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + break_flag = true; + return result; + } + + if(osf_header.bucket_str_size != encrypted_bucket_bytes.size()) { + osf_header.bucket_str_size = encrypted_bucket_bytes.size(); + } + + auto ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + break_flag = true; + return result; + } + + filepos += encrypted_bucket_bytes.size(); + data_hash_array.push_back(data_hash); + ++bucket_index; + bucket_array.clear(); + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + } + } + + + LOG(INFO) << "write real blocks done"; + + + // 3. write position_map + osf_header.position_map_filepos = filepos; + oram_ntt::position_map_t position_map_pkg; + position_map_pkg.set(position_map_array); + bytes position_map_bytes = ypc::make_bytes::for_package(position_map_pkg); + ypc::bytes encrypted_position_map_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + pub_key, position_map_bytes, ypc::utc::crypto_prefix_arbitrary, encrypted_position_map_bytes); + if (status != 0u) { + LOG(ERROR) << "encrypt_message_with_prefix fail: " + << stbox::status_string(status); + return result; + } + + ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, encrypted_position_map_bytes.data(), encrypted_position_map_bytes.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + filepos += encrypted_position_map_bytes.size(); + + + // 4. write merkle tree + osf_header.merkle_tree_filepos = filepos; + + for(int i = (1 << osf_header.level_num_L) - 2; i >= 0; --i) { + ypc::bytes k_hash = data_hash_array[i] + data_hash_array[2*i + 1]; + crypto_ptr->hash_256(k_hash, data_hash_array[i]); + + k_hash = data_hash_array[i] + data_hash_array[2*i + 2]; + crypto_ptr->hash_256(k_hash, data_hash_array[i]); + } + + for(auto &data_hash : data_hash_array) { + ret = stbox::ocall_cast(write_convert_data_structure) + (filepos, data_hash.data(), data_hash.size()); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + filepos += data_hash.size(); + } + + + // 5. update and write osf_header + osf_header.stash_filepos = filepos; + osf_header.item_num_each_batch = 0; + osf_header.item_size = 0; + + ret = stbox::ocall_cast(write_convert_data_structure) + (0, (uint8_t *)&osf_header, sizeof(osf_header)); + if (ret != stbox::stx_status::success) { + LOG(ERROR) << "write_convert_data_structure ocall fail!"; + return result; + } + + + LOG(INFO) << "convert_parse2 done"; + + return result; + } + + +protected: + ypc::data_source *m_source; +}; +''' + + with open(file_path2, "w") as file: + file.write(content) + + + + + + +if __name__ == "__main__": + + crypto = "stdeth" + index_name = "ZJHM" + mode = "debug" # or "prerelease" or "release" + + + crypto_name = get_crypto_name(crypto) + + # 根据以上变量，生成转换enclave中执行的转换算法定义文件 + file_path = os.path.join(common.sdk_dir, "example/convert/enclave/convert_parser.h") + file_path2 = os.path.join(common.sdk_dir, "example/convert/enclave2/convert_parser2.h") + + gen_convert_parser(file_path, crypto_name, index_name) + gen_convert_parser2(file_path2, crypto_name) + + build_dir = os.path.join(common.sdk_dir, 'build', mode) + + # 必须进入项目顶层目录执行 + if(enter_directory(common.sdk_dir)): + + # 重新编译 + if compile_in_subdir(build_dir) == False: + recompile(mode) + + # 执行转换enclave并测试转换后加密数据文件是否能被正常读写 + r = run_test_convert_parser() + print(r[1]) \ No newline at end of file diff --git a/test/integrate/generate_oram.py b/test/integrate/generate_oram.py new file mode 100644 index 00000000..a85b2dc8 --- /dev/null +++ b/test/integrate/generate_oram.py @@ -0,0 +1,173 @@ +import os +import common +import subprocess + +# 检查目录是否存在 +def check_directory(directory_path): + if os.path.exists(directory_path): + return True + else: + print(f"\"{directory_path}\" does not exist.") + return False + +# 进入目录 +def enter_directory(directory_path): + if check_directory(directory_path): + os.chdir(directory_path) + return True + else: + return False + +def compile_in_subdir(build_dir): + # build_dir + makefile_path = os.path.join(build_dir, 'Makefile') + if not os.path.isfile(makefile_path): + print(f"Error: makefile not found in {build_dir}") + return False + + # 执行编译命令 + try: + result = subprocess.run(['make', '-j8'], cwd = build_dir, check = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + print(result.stdout.decode('utf-8')) + return True + except subprocess.CalledProcessError as e: + print(f"Error: {e.stderr.decode('utf-8')}") + return False + +def recompile(mode): + cmd = os.path.join(common.sdk_dir, "./build.sh compile-project {} -j8").format(mode) + output = common.execute_cmd(cmd) + return [cmd, output] + +def run_test_findperson_oram(): + cmd = 'python3 {}/test/integrate/test_findperson_oram.py'.format(common.sdk_dir) + output = common.execute_cmd(cmd) + return [cmd, output] + + +def gen_person_reader_oram(file_path, index_name): + content = '''#include "person_reader_oram.h" +#include "ypc/common/limits.h" +#include "../common.h" +#include +#include +#include +#include +#include + +void *create_item_reader(const char *file_path, int len) { + + file_t *f = new file_t(); + f->open_for_read(file_path); + f->reset_read_item(); + return f; +} + +int reset_for_read(void *handle) { + if (!handle) { + return -1; + } + file_t *f = (file_t *)handle; + f->reset_read_item(); + return 0; +} + +int read_item_data(void *handle, char *buf, int *len) { + if (!handle) { + return -1; + } + if (!buf) { + return -1; + } + file_t *f = (file_t *)handle; + + ypc::memref r; + bool t = f->next_item(r); + if (t) { + memcpy(buf, r.data(), r.size()); + *len = r.size(); + r.dealloc(); + return 0; + } else { + *len = 0; + return -1; + } + + return 0; +} + +int close_item_reader(void *handle) { + file_t *f = (file_t *)handle; + f->close(); + return 0; +} + +uint64_t get_item_number(void *handle) { + file_t *f = (file_t *)handle; + return f->item_number(); +} + +int get_item_index_field(void *handle, char *buf, int *len) { + if (!handle) { + return -1; + } + if (!buf) { + return -1; + } + file_t *f = (file_t *)handle; + + ypc::memref r; + bool t = f->next_item(r); + if (t) { + typedef typename ypc::cast_obj_to_package::type pkg_t; + auto item_pkg = + ypc::make_package::from_bytes(ypc::bytes(r.data(), r.size())); + std::string index_field = item_pkg.get<''' + + content = content + index_name + + content = content + '''>(); + memcpy(buf, index_field.c_str(), index_field.size()); + *len = index_field.size(); + + r.dealloc(); + return 0; + } else { + *len = 0; + return -1; + } + + return 0; +}''' + + with open(file_path, "w") as file: + file.write(content) + + + + +if __name__ == "__main__": + + index_name = "ZJHM" + mode = "debug" # or "prerelease" or "release" + + file_path = os.path.join(common.sdk_dir, "example/oram_personlist/plugin/person_reader_oram.cpp") + + gen_person_reader_oram(file_path, index_name) + + build_dir = os.path.join(common.sdk_dir, 'build', mode) + + # 必须进入项目顶层目录执行 + if(enter_directory(common.sdk_dir)): + + # 重新编译 + if compile_in_subdir(build_dir) == False: + recompile(mode) + + # 执行转换enclave并测试转换后加密数据文件是否能被正常读写 + r = run_test_findperson_oram() + print(r[1]) + + + + diff --git a/test/integrate/job_step.py b/test/integrate/job_step.py index 7b229fdc..4c584b97 100644 --- a/test/integrate/job_step.py +++ b/test/integrate/job_step.py @@ -26,6 +26,17 @@ def seal_data(crypto, data_url, plugin_url, sealed_data_url, sealed_output, data "use-publickey-file": data_key_file } return common.fid_data_provider(**param) + + def oram_seal_data(crypto, data_url, plugin_url, sealed_data_url, sealed_output, data_key_file): + param = { + "crypto": crypto, + "data-url": data_url, + "plugin-path": plugin_url, + "sealed-data-url": sealed_data_url, + "output": sealed_output, + "use-publickey-file": data_key_file + } + return common.fid_oram_data_provider(**param) def get_first_key(crypto): keys = common.fid_keymgr_list(crypto) @@ -47,6 +58,19 @@ def read_data_hash(fp): if l.startswith("data_id"): ks = l.split("=") return ks[1].strip() + + # TODO:单独写一个可执行程序可以读取加密文件的root hash + def get_root_hash(sealed_data_url, root_hash_url): + param = { + "data-url": sealed_data_url, + "output": root_hash_url + } + return common.fid_get_root_hash(**param) + + def read_root_hash(fp): + with open(fp) as f: + root_hash = f.read() + return root_hash def read_parser_hash(parser_url): param = { @@ -127,6 +151,82 @@ def fid_analyzer(shukey_json, rq_forward_json, enclave_hash, input_data, parser_ # result is not json format with open(parser_output_file) as of: return of.readlines() + + def fid_convert_analyzer(shukey_json, data_shukey_json, rq_forward_json, enclave_hash, input_data, parser_url, dian_pkey, model, crypto, param_json, allowances, parser_input_file, parser_output_file): + parser_input = { + "shu_info": { + "shu_pkey": shukey_json["public-key"], + "encrypted_shu_skey": rq_forward_json["encrypted_skey"], + "shu_forward_signature": rq_forward_json["forward_sig"], + "enclave_hash": enclave_hash + }, + "input_data": input_data, + "parser_path": parser_url, + "keymgr_path": common.kmgr_enclave[crypto], + "parser_enclave_hash": enclave_hash, + "dian_pkey": dian_pkey, + "model": model, + "param": { + "crypto": crypto, + "param_data": param_json["encrypted-input"], + "public-key": data_shukey_json["public-key"], + } + } + if allowances: + parser_input['param']['allowances'] = allowances + with open(parser_input_file, "w") as of: + json.dump(parser_input, of) + param = { + "input": parser_input_file, + "output": parser_output_file + } + r = common.fid_analyzer(**param) + print("done fid_analyzer with cmd: {}".format(r[0])) + try: + with open(parser_output_file) as of: + return json.load(of) + except Exception as e: + # result is not json format + with open(parser_output_file) as of: + return of.readlines() + + def fid_oram_analyzer(shukey_json, rq_forward_json, enclave_hash, input_data, parser_url, dian_pkey, model, crypto, param_json, allowances, parser_input_file, parser_output_file): + parser_input = { + "shu_info": { + "shu_pkey": shukey_json["public-key"], + "encrypted_shu_skey": rq_forward_json["encrypted_skey"], + "shu_forward_signature": rq_forward_json["forward_sig"], + "enclave_hash": enclave_hash + }, + "input_data": input_data, + "parser_path": parser_url, + "keymgr_path": common.kmgr_enclave[crypto], + "parser_enclave_hash": enclave_hash, + "dian_pkey": dian_pkey, + "model": model, + "param": { + "crypto": crypto, + "param_data": param_json["encrypted-input"], + "public-key": shukey_json["public-key"], + } + } + if allowances: + parser_input['param']['allowances'] = allowances + with open(parser_input_file, "w") as of: + json.dump(parser_input, of) + param = { + "input": parser_input_file, + "output": parser_output_file + } + r = common.fid_oram_analyzer(**param) + print("done fid_analyzer with cmd: {}".format(r[0])) + try: + with open(parser_output_file) as of: + return json.load(of) + except Exception as e: + # result is not json format + with open(parser_output_file) as of: + return of.readlines() def decrypt_result(crypto, encrypted_result, shukey_file, decrypted_result): param = { diff --git a/test/integrate/test_convert_parser.py b/test/integrate/test_convert_parser.py new file mode 100644 index 00000000..97af5cb0 --- /dev/null +++ b/test/integrate/test_convert_parser.py @@ -0,0 +1,53 @@ +from classic_job_oram import classic_job_oram +from classic_job_convert import classic_job_convert +import os +import common +import sys +import random +import re + +def gen_personlist(**kwargs): + cmd = os.path.join(common.bin_dir, "./personlist_gen_oram") + output = common.execute_cmd(cmd) + return [cmd, output] + + +if __name__ == "__main__": + name = "convert_person" + gen_personlist() + + crypto = "stdeth" + # crypto = "gmssl" + data = "person_list" + convert_parser = os.path.join(common.lib_dir, "convert_sealed_file.signed.so") + convert_parser2 = os.path.join(common.lib_dir, "convert_sealed_file2.signed.so") + oram_parser = os.path.join(common.lib_dir, "person_first_match_oram.signed.so") + plugin = os.path.join( + common.lib_dir, "libperson_reader{}.so".format(common.debug_postfix())) + + # con_read_num必须>=3，前两次转换成ORAM密封格式 + con_read_num = 6 + + cj = classic_job_convert(crypto, name, data, convert_parser, convert_parser2, oram_parser, plugin, con_read_num, { + 'request-use-js': True, + 'remove-files': True if len(sys.argv) < 2 else False, + }) + + id = 421003198607262336 + result = [] + for i in range(con_read_num): + ran_i = random.randint(0, 7970 - 1) + id_str = str(id + ran_i) + input_param = "\"[{\\\"type\\\":\\\"string\\\",\\\"value\\\":\\\"" + id_str + "\\\"}]\"" + + cj.input = input_param + cj.run() + + if(cj.read_count > 2): + result.append(cj.result) + + matches = re.findall(r"\d+", cj.result[0]) + if matches[0] != str(id_str): + print("not find target row!") + break + print(result) \ No newline at end of file diff --git a/test/integrate/test_findperson_oram.py b/test/integrate/test_findperson_oram.py new file mode 100644 index 00000000..262bacf2 --- /dev/null +++ b/test/integrate/test_findperson_oram.py @@ -0,0 +1,48 @@ +from classic_job_oram import classic_job_oram +import os +import common +import sys +import random +import re + +def gen_personlist(**kwargs): + cmd = os.path.join(common.bin_dir, "./personlist_gen_oram") + output = common.execute_cmd(cmd) + return [cmd, output] + + +if __name__ == "__main__": + name = "findperson_oram" + gen_personlist() + + crypto = "stdeth" + data = "person_list" + parser = os.path.join(common.lib_dir, "person_first_match_oram.signed.so") + plugin = os.path.join( + common.lib_dir, "libperson_reader_oram{}.so".format(common.debug_postfix())) + + con_read_num = 5 + + cj = classic_job_oram(crypto, name, data, parser, plugin, con_read_num, { + 'request-use-js': True, + 'remove-files': True if len(sys.argv) < 2 else False, + }) + + id = 421003198607262336 + result = [] + for i in range(con_read_num): + ran_i = random.randint(0, 7970 - 1) + id_str = str(id + ran_i) + # id_str = str(id + i) + input_param = "\"[{\\\"type\\\":\\\"string\\\",\\\"value\\\":\\\"" + id_str + "\\\"}]\"" + cj.input = input_param + cj.run() + result.append(cj.result) + # print("input_param is : ", input_param) + # print("result is : ", cj.result) + matches = re.findall(r"\d+", cj.result[0]) + print(str(id)) + if matches[0] != str(id_str): + print("not find target row!") + break + print(result) \ No newline at end of file diff --git a/toolkit/analyzer/CMakeLists.txt b/toolkit/analyzer/CMakeLists.txt index 5813ba0d..d034e8fa 100644 --- a/toolkit/analyzer/CMakeLists.txt +++ b/toolkit/analyzer/CMakeLists.txt @@ -21,3 +21,26 @@ install(TARGETS fid_analyzer DESTINATION "${bin_install_dir}") AddClangTidy(fid_analyzer) EnableCoverage(fid_analyzer) + +add_executable(fid_oram_analyzer oram_analyzer.cpp + sgx_bridge.cpp + parsers/oram_parser.cpp + ) + +target_link_libraries(fid_oram_analyzer + stbox_common + core + core_parser_module + keymgr_module + keymgr_utils + glog + ) +target_include_directories(fid_oram_analyzer PRIVATE + "${PROJECT_SOURCE_DIR}/toolkit/analyzer/" + "$" + ) + +install(TARGETS fid_oram_analyzer + DESTINATION "${bin_install_dir}") +AddClangTidy(fid_oram_analyzer) +EnableCoverage(fid_oram_analyzer) \ No newline at end of file diff --git a/toolkit/analyzer/iodef.h b/toolkit/analyzer/iodef.h index 340e8bfb..f223e014 100644 --- a/toolkit/analyzer/iodef.h +++ b/toolkit/analyzer/iodef.h @@ -2,9 +2,11 @@ #include "ypc/core/byte.h" #include "ypc/core/ntjson.h" #include "ypc/corecommon/nt_cols.h" +#include "ypc/corecommon/oram_types.h" #include "ypc/corecommon/package.h" using ntt = ypc::nt; +using oram_ntt = ypc::oram::nt; define_nt(input_data_url, std::string); define_nt(input_data_hash, ypc::bytes); diff --git a/toolkit/analyzer/oram_analyzer.cpp b/toolkit/analyzer/oram_analyzer.cpp new file mode 100644 index 00000000..2184ba44 --- /dev/null +++ b/toolkit/analyzer/oram_analyzer.cpp @@ -0,0 +1,95 @@ +#include "iodef.h" +#include "ypc/core/version.h" +#include "sgx_bridge.h" +#include "ypc/core/configuration.h" +#include "ypc/core/ntobject_file.h" +#include "ypc/core/oram_sealed_file.h" +#include "ypc/stbox/stx_status.h" +#include +#include +#include +#include +#include +#include +#include + +using stx_status = stbox::stx_status; +using namespace ypc; + +boost::program_options::variables_map parse_command_line(int argc, + char *argv[]) { + namespace bp = boost::program_options; + bp::options_description all("YeeZ Privacy Analyzer options"); + + // clang-format off + all.add_options() + ("help", "help message") + ("version", "show version") + ("input", bp::value(), "input parameters JSON file") + ("output", bp::value(), "output result JSON file") + ("gen-example-input", bp::value(), "generate example input parameters JSON file"); + // clang-format on + + boost::program_options::variables_map vm; + boost::program_options::store( + boost::program_options::parse_command_line(argc, argv, all), vm); + + if (vm.count("help") != 0u) { + std::cout << all << std::endl; + exit(-1); + } + if (vm.count("version") != 0u) { + std::cout << ypc::get_ypc_version() << std::endl; + exit(-1); + } + if (vm.count("gen-example-input") != 0u) { + input_param_t example; + ypc::ntjson::to_json_file(example, + vm["gen-example-input"].as()); + exit(-1); + } + return vm; +} + +int main(int argc, char *argv[]) { + + // google::InitGoogleLogging(argv[0]); + // google::InstallFailureSignalHandler(); + + boost::program_options::variables_map vm; + try { + vm = parse_command_line(argc, argv); + } catch (...) { + std::cerr << "invalid cmd line parameters!" << std::endl; + return -1; + } + if (vm.count("input") == 0u) { + std::cerr << "input not specified" << std::endl; + return -1; + } + + if (vm.count("output") == 0u) { + std::cerr << "output not specified" << std::endl; + return -1; + } + + input_param_t input_param = + ypc::ntjson::from_json_file(vm["input"].as()); + + o_parser = std::make_shared(input_param); + std::cout << "start to parse" << std::endl; + o_parser->oram_parse(); + + std::string output_fp = vm["output"].as(); + try { + std::ofstream os(output_fp, std::ios::out | std::ios::binary); + const std::string &res = o_parser->get_result_str(); + os.write(res.data(), res.size()); + } catch (const std::exception &e) { + std::cerr << "cannot open " << output_fp << std::endl; + return 1; + } + + return 0; +} + diff --git a/toolkit/analyzer/parsers/oram_parser.cpp b/toolkit/analyzer/parsers/oram_parser.cpp new file mode 100644 index 00000000..d5a44a6f --- /dev/null +++ b/toolkit/analyzer/parsers/oram_parser.cpp @@ -0,0 +1,559 @@ +#include "parsers/oram_parser.h" +#include "ypc/common/access_policy.h" +#include "ypc/corecommon/nt_cols.h" +#include "ypc/core/status.h" +#include "ypc/core/ntjson.h" +#include "ypc/corecommon/package.h" +#include + +#define FIND_DATA_SOURCE \ + auto hash = ypc::bytes(data_hash, hash_size); \ + if (m_data_sources.find(hash) == m_data_sources.end()) \ + { \ + LOG(ERROR) << "data with hash: " << hash << " not found"; \ + return stbox::stx_status::data_source_not_found; \ + } \ + auto sosf = m_data_sources[hash]; + +#define DOWNLOAD_OCALL_RETURN(buffer, buffer_size) \ + if (ret) \ + { \ + *buffer = buffer##_mem.data(); \ + *buffer_size = buffer##_mem.size(); \ + return stbox::stx_status::success; \ + } \ + return stbox::stx_status::oram_sealed_file_error; + +#define DOWNLOAD_OCALL(name, buffer, buffer_size) \ + uint32_t oram_parser::name##_OCALL(const uint8_t *data_hash, uint32_t hash_size, \ + uint8_t **buffer, uint32_t *buffer_size) \ + { \ + FIND_DATA_SOURCE \ + ypc::memref buffer##_mem; \ + bool ret = sosf->name(buffer##_mem); \ + DOWNLOAD_OCALL_RETURN(buffer, buffer_size) \ + } + +#define DOWNLOAD_LEAF_OCALL(name, buffer, buffer_size) \ + uint32_t oram_parser::name##_OCALL(const uint8_t *data_hash, uint32_t hash_size, \ + uint32_t leaf, uint8_t **buffer, uint32_t *buffer_size) \ + { \ + FIND_DATA_SOURCE \ + ypc::memref buffer##_mem; \ + bool ret = sosf->name(leaf, buffer##_mem); \ + DOWNLOAD_OCALL_RETURN(buffer, buffer_size) \ + } + +#define NOMAL_RETURN \ + if (ret) \ + { \ + return stbox::stx_status::success; \ + } \ + return stbox::stx_status::oram_sealed_file_error; + +#define UPDATE_OCALL(name, buffer, buffer_size) \ + uint32_t oram_parser::name##_OCALL(const uint8_t *data_hash, uint32_t hash_size, \ + const uint8_t *buffer, uint32_t buffer_size) \ + { \ + FIND_DATA_SOURCE \ + bool ret = sosf->name(buffer, buffer_size); \ + NOMAL_RETURN \ + } + +#define UPDATE_LEAF_OCALL(name, buffer, buffer_size) \ + uint32_t oram_parser::name##_OCALL(const uint8_t *data_hash, uint32_t hash_size, \ + uint32_t leaf, const uint8_t *buffer, uint32_t buffer_size) \ + { \ + FIND_DATA_SOURCE \ + bool ret = sosf->name(leaf, buffer, buffer_size); \ + NOMAL_RETURN \ + } + +DOWNLOAD_OCALL(download_position_map, position_map, len) +DOWNLOAD_OCALL(download_stash, stash, len) +DOWNLOAD_LEAF_OCALL(download_path, encrpypted_path, len) +DOWNLOAD_LEAF_OCALL(download_merkle_hash, encrpypted_path, len) +UPDATE_OCALL(update_position_map, position_map, len) +UPDATE_OCALL(update_stash, stash, len) +UPDATE_LEAF_OCALL(upload_path, encrpypted_path, len) +UPDATE_LEAF_OCALL(update_merkle_hash, merkle_hash, len) + +uint32_t oram_parser::download_oram_params_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_num, uint32_t *bucket_num_N, uint8_t *level_num_L, + uint32_t *bucket_str_size, uint32_t *batch_str_size) { + FIND_DATA_SOURCE + bool ret = sosf->download_oram_params(block_num, bucket_num_N, level_num_L, bucket_str_size, batch_str_size); + NOMAL_RETURN +} + +uint32_t oram_parser::get_block_id_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_id, + const uint8_t *param_hash, uint32_t param_hash_size) { + + FIND_DATA_SOURCE + ypc::bytes item_index_field_hash(param_hash, param_hash_size); + bool ret = sosf->get_block_id(item_index_field_hash, block_id); + NOMAL_RETURN +} + + + + +oram_parser::oram_parser(const input_param_t ¶m) : m_param(param) {} + +oram_parser::~oram_parser() = default; + +ypc::bytes construct_access_control_policy() { + using ntt = ypc::nt; + ntt::access_list_package_t alp; + alp.set(ypc::utc::access_policy_blacklist); + alp.set(std::vector()); + return ypc::make_bytes::for_package(alp); +} + +uint32_t oram_parser::oram_parse() { + auto parser_enclave_path = m_param.get(); +#ifdef DEBUG + LOG(INFO) << "parser enclave path: " << parser_enclave_path; +#endif + auto keymgr_enclave_path = m_param.get(); + m_parser = + std::make_shared(parser_enclave_path.c_str()); +#ifdef DEBUG + LOG(INFO) << "keymgr enclave path: " << keymgr_enclave_path; +#endif + m_keymgr = std::make_shared(keymgr_enclave_path.c_str()); + + ypc::bytes policy = construct_access_control_policy(); + m_keymgr->set_access_control_policy(policy); + LOG(INFO) << "initializing parser/keymgr module done"; + + auto epkey = m_param.get(); + auto ehash = m_param.get(); + auto shu_skey = m_param.get().get(); + auto shu_forward_sig = + m_param.get().get(); + + uint32_t ret = 0; + if (!shu_skey.empty()) { + LOG(INFO) << "keymgr_enclave_path: " << keymgr_enclave_path; + ret = m_keymgr->forward_private_key( + shu_skey.data(), shu_skey.size(), epkey.data(), epkey.size(), + ehash.data(), ehash.size(), shu_forward_sig.data(), + shu_forward_sig.size()); + if (ret != 0u) { + LOG(ERROR) << "forward_message got error " << ypc::status_string(ret); + return ret; + } + } + m_ptype.value = m_parser->get_parser_type(); + ypc::bytes actual_hash; + ret = m_parser->get_enclave_hash(actual_hash); + if (ret != 0u) { + LOG(ERROR) << "get_enclave_hash got error " << ypc::status_string(ret); + return ret; + } + + if (actual_hash != ehash) { + LOG(ERROR) << "parser hash is " << actual_hash << ", expect " << ehash; + return ypc::parser_return_wrong_data_hash; + } + + ret = feed_datasource(); + if (ret != 0u) { + LOG(ERROR) << "feed_datasource got error " << ypc::status_string(ret); + return ret; + } + + ret = feed_model(); + if (ret != 0u) { + LOG(ERROR) << "feed_model got error " << ypc::status_string(ret); + return ret; + } + + ret = m_parser->begin_parse_data_item(); + if (ret != stx_status::success) { + LOG(ERROR) << "begin_parse_data_item, got error: " + << ypc::status_string(ret); + return ret; + } + auto param_var = m_param.get(); + typename ypc::cast_obj_to_package::type param_pkg = param_var; + auto param_bytes = ypc::make_bytes::for_package(param_pkg); + ret = m_parser->parse_data_item(param_bytes.data(), param_bytes.size()); + if (ret != 0u) { + LOG(ERROR) << "parse_data_item, got error: " << ypc::status_string(ret); + return ret; + } + + ret = m_parser->end_parse_data_item(); + if (ret != stx_status::success) { + LOG(ERROR) << "end_parse_data_item, got error: " << ypc::status_string(ret); + } + + if (ret != 0u) { + LOG(ERROR) << "do_parse got error " << ypc::status_string(ret); + return ret; + } + LOG(INFO) << "parse done"; + + ypc::bytes res; + ret = m_parser->get_analyze_result(res); + if (ret != 0u) { + LOG(ERROR) << "get_analyze_result got error " << ypc::status_string(ret); + return ret; + } + ret = dump_result(res); + if (ret != 0u) { + LOG(ERROR) << "dump_result got error " << ypc::status_string(ret); + return ret; + } + + return ypc::success; +} + +uint32_t oram_parser::dump_result(const ypc::bytes &res) { + if (m_ptype.d.result_type == ypc::utc::onchain_result_parser) { + auto pkg = + ypc::make_package::from_bytes(res); + typename ypc::cast_package_to_obj::type p = + pkg; + m_result_str = ypc::ntjson::to_json(p); + } else if (m_ptype.d.result_type == ypc::utc::offchain_result_parser) { + auto pkg = + ypc::make_package::from_bytes(res); + typename ypc::cast_package_to_obj::type p = + pkg; + m_result_str = ypc::ntjson::to_json(p); + + } else if (m_ptype.d.result_type == ypc::utc::local_result_parser) { + m_result_str = std::string((const char *)res.data(), res.size()); + } else if (m_ptype.d.result_type == ypc::utc::forward_result_parser) { + auto pkg = ypc::make_package::type>::from_bytes(res); + + typename ypc::cast_package_to_obj::type p = pkg; + m_result_str = ypc::ntjson::to_json(p); + } else { + return ypc::parser_unknown_result; + } + return ypc::success; +} + +uint32_t oram_parser::feed_datasource() { + auto input_data_var = m_param.get(); + if (m_ptype.d.data_source_type == ypc::utc::noinput_datasource_parser) { + return ypc::success; + } + + if (m_ptype.d.data_source_type == ypc::utc::oram_sealed_datasource_parser) { + if (input_data_var.empty()) { + LOG(ERROR) << "missing input, require one input data source"; + return ypc::parser_missing_input; + } if (input_data_var.size() > 1) { + LOG(WARNING) << "only need 1 input, ignore other inputs"; + } + } + + if (m_ptype.d.data_source_type == ypc::utc::single_sealed_datasource_parser) { + if (input_data_var.empty()) { + LOG(ERROR) << "missing input, require one input data source"; + return ypc::parser_missing_input; + } if (input_data_var.size() > 1) { + LOG(WARNING) << "only need 1 input, ignore other inputs"; + } + } + if (m_ptype.d.data_source_type == ypc::utc::multi_sealed_datasource_parser) { + if (input_data_var.empty()) { + LOG(ERROR) << "missing input, require at least one input data source"; + return ypc::parser_missing_input; + } + } + + auto epkey = m_param.get(); + auto param_var = m_param.get(); + std::vector all_data_info; + for (auto item : input_data_var) { + auto url = item.get(); + auto data_hash = item.get(); + auto sosf = std::make_shared(url); + + m_data_sources.insert(std::make_pair(data_hash, sosf)); + sosf->open_for_write(); + + auto shu = item.get(); + auto shu_skey = shu.get(); + auto shu_forward_sig = shu.get(); + auto target_enclave_hash = shu.get(); + auto ret = m_keymgr->forward_private_key( + shu_skey.data(), shu_skey.size(), epkey.data(), epkey.size(), + target_enclave_hash.data(), target_enclave_hash.size(), + shu_forward_sig.data(), shu_forward_sig.size()); + if (ret != 0u) { + LOG(ERROR) << "forward_message got error " << ypc::status_string(ret); + return ret; + } + + ntt::oram_sealed_data_info_t data_info; + data_info.set( + data_hash, shu.get(), item.get(), + param_var.get(), param_var.get()); + all_data_info.push_back(data_info.make_copy()); + + } + + ypc::bytes data_info_bytes; + if (m_ptype.d.data_source_type == ypc::utc::oram_sealed_datasource_parser) { + if (all_data_info.empty()) { + LOG(ERROR) << "cannot get input data info"; + return ypc::parser_missing_input; + } + + typename ypc::cast_obj_to_package::type oram = + all_data_info[0]; + data_info_bytes = ypc::make_bytes::for_package(oram); + } + + if (m_ptype.d.data_source_type == ypc::utc::single_sealed_datasource_parser) { + if (all_data_info.empty()) { + LOG(ERROR) << "cannot get input data info"; + return ypc::parser_missing_input; + } + + typename ypc::cast_obj_to_package::type single = + all_data_info[0]; + data_info_bytes = ypc::make_bytes::for_package(single); + } + + if (m_ptype.d.data_source_type == ypc::utc::multi_sealed_datasource_parser) { + data_info_bytes = ypc::make_bytes::for_package< + typename ypc::cast_obj_to_package::type, + ntt::oram_sealed_data_info_vector>(all_data_info); + } + if (m_ptype.d.data_source_type == ypc::utc::raw_datasource_parser) { + data_info_bytes = all_data_info[0].get(); + } + + auto ret = m_parser->init_data_source(data_info_bytes); + if (ret != 0u) { + LOG(ERROR) << "init_data_source got error " << ypc::status_string(ret); + return ret; + } + return ypc::success; +} + +uint32_t oram_parser::feed_model() { + if (m_ptype.d.has_model == ypc::utc::no_model_parser) { + return ypc::success; + } + + auto model = m_param.get(); + ypc::cast_obj_to_package::type pkg = model; + + auto model_bytes = ypc::make_bytes::for_package(pkg); + uint32_t ret = m_parser->init_model(model_bytes); + if (ret != 0u) { + LOG(ERROR) << "init_model got error: " << ypc::status_string(ret); + return ret; + } + + return ypc::success; +} +uint32_t oram_parser::feed_param() { return ypc::success; } + +// uint32_t oram_parser::download_oram_params_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint32_t *block_num, uint32_t *bucket_num_N, uint8_t *level_num_L, +// uint32_t *bucket_str_size, uint32_t *batch_str_size) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// bool ret = sosf->download_oram_params(block_num, bucket_num_N, level_num_L, bucket_str_size, batch_str_size); +// if(ret) { +// return stbox::stx_status::success; +// } + +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::get_block_id_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint32_t *block_id, +// const uint8_t *param_hash, uint32_t param_hash_size) { + +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; + +// sosf->reset(); +// ypc::bytes item_index_field_hash(param_hash, param_hash_size); +// bool ret = sosf->get_block_id(item_index_field_hash, block_id); +// if(ret) { +// return stbox::stx_status::success; +// } + +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::download_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint8_t ** position_map, uint32_t *len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// ypc::memref posmap; +// bool ret = sosf->download_position_map(posmap); +// if(ret) { +// *position_map = posmap.data(); +// *len = posmap.size(); +// return stbox::stx_status::success; +// } + +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::update_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// const uint8_t * position_map, uint32_t len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// bool ret = sosf->update_position_map(position_map, len); +// if(ret) { +// return stbox::stx_status::success; +// } + + +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::download_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint32_t leaf, uint8_t ** encrpypted_path, uint32_t *len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// ypc::memref en_path; +// bool ret = sosf->download_path(leaf, en_path); +// if(ret) { +// *encrpypted_path = en_path.data(); +// *len = en_path.size(); +// return stbox::stx_status::success; +// } + + +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::download_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint8_t ** stash, uint32_t *len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// ypc::memref st; +// bool ret = sosf->download_stash(st); +// if(ret) { +// *stash = st.data(); +// *len = st.size(); +// return stbox::stx_status::success; +// } + +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::update_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// const uint8_t * stash, uint32_t len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } + +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// bool ret = sosf->update_stash(stash, len); +// if(ret) { +// return stbox::stx_status::success; +// } + +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::upload_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint32_t leaf, const uint8_t * encrpypted_path, uint32_t len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// bool ret = sosf->upload_path(leaf, encrpypted_path, len); +// if(ret) { +// return stbox::stx_status::success; +// } + +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::download_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint32_t leaf, uint8_t ** merkle_hash, uint32_t *len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// ypc::memref me_hash; +// bool ret = sosf->download_merkle_hash(leaf, me_hash); +// if(ret) { +// *merkle_hash = me_hash.data(); +// *len = me_hash.size(); +// return stbox::stx_status::success; +// } + +// return stbox::stx_status::oram_sealed_file_error; +// } + +// uint32_t oram_parser::update_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, +// uint32_t leaf, const uint8_t * merkle_hash, uint32_t len) { +// auto hash = ypc::bytes(data_hash, hash_size); +// if (m_data_sources.find(hash) == m_data_sources.end()) { +// LOG(ERROR) << "data with hash: " << hash << " not found"; +// return stbox::stx_status::data_source_not_found; +// } +// auto sosf = m_data_sources[hash]; +// sosf->reset(); + +// bool ret = sosf->update_merkle_hash(leaf, merkle_hash, len); +// if(ret) { +// return stbox::stx_status::success; +// } + +// return stbox::stx_status::oram_sealed_file_error; +// } \ No newline at end of file diff --git a/toolkit/analyzer/parsers/oram_parser.h b/toolkit/analyzer/parsers/oram_parser.h new file mode 100644 index 00000000..680aff1d --- /dev/null +++ b/toolkit/analyzer/parsers/oram_parser.h @@ -0,0 +1,68 @@ +#pragma once +#include "ypc/core/oram_sealed_file.h" +#include "iodef.h" +#include "ypc/common/parser_type.h" +#include "ypc/core/sgx/parser_sgx_module.h" +#include "ypc/keymgr/default/keymgr_sgx_module.h" + + +class oram_parser { +public: + oram_parser(const input_param_t ¶m); + + virtual ~oram_parser(); + + virtual uint32_t oram_parse(); + + virtual uint32_t download_oram_params_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_num, uint32_t *bucket_num_N, + uint8_t *level_num_L, uint32_t *bucket_str_size, uint32_t *batch_str_size); + + virtual uint32_t get_block_id_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_id, + const uint8_t *param_hash, uint32_t param_hash_size); + + virtual uint32_t download_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t ** position_map, uint32_t *len); + + virtual uint32_t update_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, + const uint8_t * position_map, uint32_t len); + + virtual uint32_t download_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t ** encrpypted_path, uint32_t *len); + + virtual uint32_t download_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t ** stash, uint32_t *len); + virtual uint32_t update_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + const uint8_t * stash, uint32_t len); + + virtual uint32_t upload_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, const uint8_t * encrpypted_path, uint32_t len); + + virtual uint32_t download_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t ** merkle_hash, uint32_t *len); + + virtual uint32_t update_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, const uint8_t * merkle_hash, uint32_t len); + + + inline std::shared_ptr keymgr() const { return m_keymgr; } + + inline const std::string &get_result_str() const { return m_result_str; } + +protected: + uint32_t feed_datasource(); + uint32_t feed_model(); + uint32_t feed_param(); + uint32_t dump_result(const ypc::bytes &res); + +protected: + input_param_t m_param; + ypc::utc::parser_type_t m_ptype{}; + + std::shared_ptr m_parser; + std::shared_ptr m_keymgr; + std::unordered_map> + m_data_sources; + std::string m_result_str; +}; \ No newline at end of file diff --git a/toolkit/analyzer/parsers/parser.cpp b/toolkit/analyzer/parsers/parser.cpp index 9e65596e..4c8b0ac3 100644 --- a/toolkit/analyzer/parsers/parser.cpp +++ b/toolkit/analyzer/parsers/parser.cpp @@ -87,6 +87,7 @@ uint32_t parser::parse() { } auto param_var = m_param.get(); typename ypc::cast_obj_to_package::type param_pkg = param_var; + auto param_data = param_pkg.get(); auto param_bytes = ypc::make_bytes::for_package(param_pkg); ret = m_parser->parse_data_item(param_bytes.data(), param_bytes.size()); if (ret != 0u) { @@ -178,6 +179,8 @@ uint32_t parser::feed_datasource() { m_data_sources.insert(std::make_pair(data_hash, ssf)); ssf->reset_read(); + m_oram_sealed_file_path = url + ".oram"; + auto shu = item.get(); auto shu_skey = shu.get(); auto shu_forward_sig = shu.get(); @@ -264,3 +267,65 @@ uint32_t parser::next_data_batch(const uint8_t *data_hash, uint32_t hash_size, void parser::free_data_batch(uint8_t *data) { delete[] data; } + +uint32_t parser::write_convert_data_structure(int64_t filepos, const uint8_t * convert_data_bytes, uint32_t len) { + + std::fstream m_oram_sealed_file = std::fstream(m_oram_sealed_file_path, std::ios::in | std::ios::out | std::ios::binary); + if(!m_oram_sealed_file.is_open()) { + LOG(ERROR) << "Failed to create oram sealed file: " + m_oram_sealed_file_path; + return stbox::stx_status::convert_parser_error; + } + + try { + m_oram_sealed_file.seekp(filepos, m_oram_sealed_file.beg); + m_oram_sealed_file.write((char *)convert_data_bytes, len); + } catch (const std::exception &e) { + LOG(ERROR) << "write_convert_data_structure got error: " << e.what(); + return stbox::stx_status::convert_parser_error; + } + + m_oram_sealed_file.close(); + + return stbox::stx_status::success; +} + + + + +uint32_t parser::download_convert_params_ocall(uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size) { + + std::fstream m_oram_sealed_file = std::fstream(m_oram_sealed_file_path, std::ios::in | std::ios::binary); + if(!m_oram_sealed_file.is_open()) { + LOG(ERROR) << "Failed to create oram sealed file: " + m_oram_sealed_file_path; + return stbox::stx_status::convert_parser_error; + } + + ypc::oram::header osf_header{}; + + try { + m_oram_sealed_file.seekg(0, m_oram_sealed_file.beg); + m_oram_sealed_file.read((char *)&osf_header, sizeof(osf_header)); + + *block_num = osf_header.block_num; + *oram_tree_filepos = osf_header.oram_tree_filepos; + *item_num_each_batch = osf_header.item_num_each_batch; + *item_size = osf_header.item_size; + + + LOG(INFO) << "osf_header.block_num = " << osf_header.block_num; + LOG(INFO) << "osf_header.oram_tree_filepos = " << osf_header.oram_tree_filepos; + LOG(INFO) << "osf_header.item_num_each_batch = " << osf_header.item_num_each_batch; + LOG(INFO) << "osf_header.item_size = " << osf_header.item_size; + + + } catch (const std::exception &e) { + LOG(ERROR) << "download_convert_params_ocall got error: " << e.what(); + return stbox::stx_status::convert_parser_error; + } + + m_oram_sealed_file.close(); + + return stbox::stx_status::success; +} + diff --git a/toolkit/analyzer/parsers/parser.h b/toolkit/analyzer/parsers/parser.h index cfcdaefb..b4f664ee 100644 --- a/toolkit/analyzer/parsers/parser.h +++ b/toolkit/analyzer/parsers/parser.h @@ -15,6 +15,11 @@ class parser { virtual uint32_t parse(); + virtual uint32_t write_convert_data_structure(int64_t filepos, const uint8_t * convert_data_bytes, uint32_t len); + + virtual uint32_t download_convert_params_ocall(uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size); + virtual uint32_t next_data_batch(const uint8_t *data_hash, uint32_t hash_size, uint8_t **data, uint32_t *len); virtual void free_data_batch(uint8_t *data); @@ -38,4 +43,5 @@ class parser { std::unordered_map> m_data_sources; std::string m_result_str; + std::string m_oram_sealed_file_path; }; diff --git a/toolkit/analyzer/sgx_bridge.cpp b/toolkit/analyzer/sgx_bridge.cpp index 01a3caa0..9a214490 100644 --- a/toolkit/analyzer/sgx_bridge.cpp +++ b/toolkit/analyzer/sgx_bridge.cpp @@ -3,6 +3,7 @@ using stx_status = stbox::stx_status; std::shared_ptr g_parser; +std::shared_ptr o_parser; extern "C" { uint32_t km_session_request_ocall(sgx_dh_msg1_t *dh_msg1, uint32_t *session_id); @@ -18,6 +19,48 @@ uint32_t km_end_session_ocall(uint32_t session_id); uint32_t next_data_batch(const uint8_t *data_hash, uint32_t hash_size, uint8_t **data, uint32_t *len); void free_data_batch(uint8_t *data); + + + +uint32_t km_session_request_oram_ocall(sgx_dh_msg1_t *dh_msg1, uint32_t *session_id); +uint32_t km_exchange_report_oram_ocall(sgx_dh_msg2_t *dh_msg2, + sgx_dh_msg3_t *dh_msg3, uint32_t session_id); +uint32_t km_send_request_oram_ocall(uint32_t session_id, + secure_message_t *req_message, + size_t req_message_size, size_t max_payload_size, + secure_message_t *resp_message, + size_t resp_message_size); +uint32_t km_end_session_oram_ocall(uint32_t session_id); + +uint32_t download_oram_params_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_num, uint32_t *bucket_num_N, uint8_t *level_num_L, + uint32_t *bucket_str_size, uint32_t *batch_str_size); +uint32_t get_block_id_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_id, + const uint8_t *param_hash, uint32_t param_hash_size); +uint32_t download_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t ** position_map, uint32_t *len); +uint32_t update_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t * position_map, uint32_t len); +uint32_t download_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t ** encrpypted_path, uint32_t *len); +uint32_t download_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t ** stash, uint32_t *len); +uint32_t update_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t * stash, uint32_t len); +uint32_t upload_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t * encrpypted_path, uint32_t len); +uint32_t download_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t ** merkle_hash, uint32_t *len); +uint32_t update_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t * merkle_hash, uint32_t len); + + + +uint32_t write_convert_data_structure(int64_t filepos, uint8_t * convert_data_bytes, uint32_t len); +uint32_t download_convert_params_ocall(uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size); + } uint32_t km_session_request_ocall(sgx_dh_msg1_t *dh_msg1, @@ -47,3 +90,89 @@ uint32_t next_data_batch(const uint8_t *data_hash, uint32_t hash_size, } void free_data_batch(uint8_t *data) { g_parser->free_data_batch(data); } + + +uint32_t km_session_request_oram_ocall(sgx_dh_msg1_t *dh_msg1, + uint32_t *session_id) { + return o_parser->keymgr()->session_request(dh_msg1, session_id); +} +uint32_t km_exchange_report_oram_ocall(sgx_dh_msg2_t *dh_msg2, + sgx_dh_msg3_t *dh_msg3, uint32_t session_id) { + return o_parser->keymgr()->exchange_report(dh_msg2, dh_msg3, session_id); +} +uint32_t km_send_request_oram_ocall(uint32_t session_id, + secure_message_t *req_message, + size_t req_message_size, size_t max_payload_size, + secure_message_t *resp_message, + size_t resp_message_size) { + return o_parser->keymgr()->generate_response(req_message, req_message_size, + max_payload_size, resp_message, + resp_message_size, session_id); +} +uint32_t km_end_session_oram_ocall(uint32_t session_id) { + return o_parser->keymgr()->end_session(session_id); +} + +uint32_t download_oram_params_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_num, uint32_t *bucket_num_N, uint8_t *level_num_L, + uint32_t *bucket_str_size, uint32_t *batch_str_size) { + return o_parser->download_oram_params_OCALL(data_hash, hash_size, block_num, + bucket_num_N, level_num_L, bucket_str_size, batch_str_size); +} + +uint32_t get_block_id_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_id, + const uint8_t *param_hash, uint32_t param_hash_size) { + return o_parser->get_block_id_OCALL(data_hash, hash_size, block_id, param_hash, param_hash_size); +} + +uint32_t download_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t ** position_map, uint32_t *len) { + return o_parser->download_position_map_OCALL(data_hash, hash_size, position_map, len); +} + +uint32_t update_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t * position_map, uint32_t len) { + return o_parser->update_position_map_OCALL(data_hash, hash_size, position_map, len); +} + +uint32_t download_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t ** encrpypted_path, uint32_t *len) { + return o_parser->download_path_OCALL(data_hash, hash_size, leaf, encrpypted_path, len); +} + +uint32_t download_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t ** stash, uint32_t *len) { + return o_parser->download_stash_OCALL(data_hash, hash_size, stash, len); +} + +uint32_t update_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t * stash, uint32_t len) { + return o_parser->update_stash_OCALL(data_hash, hash_size, stash, len); +} + +uint32_t upload_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t * encrpypted_path, uint32_t len) { + return o_parser->upload_path_OCALL(data_hash, hash_size, leaf, encrpypted_path, len); +} + +uint32_t download_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t ** merkle_hash, uint32_t *len) { + return o_parser->download_merkle_hash_OCALL(data_hash, hash_size, leaf, merkle_hash, len); +} + +uint32_t update_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t * merkle_hash, uint32_t len) { + return o_parser->update_merkle_hash_OCALL(data_hash, hash_size, leaf, merkle_hash, len); +} + + + +uint32_t write_convert_data_structure(int64_t filepos, uint8_t * convert_data_bytes, uint32_t len) { + return g_parser->write_convert_data_structure(filepos, convert_data_bytes, len); +} + +uint32_t download_convert_params_ocall(uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size) { + return g_parser->download_convert_params_ocall(block_num, oram_tree_filepos, item_num_each_batch, item_size); +} \ No newline at end of file diff --git a/toolkit/analyzer/sgx_bridge.h b/toolkit/analyzer/sgx_bridge.h index f4c13ecd..f4f501d3 100644 --- a/toolkit/analyzer/sgx_bridge.h +++ b/toolkit/analyzer/sgx_bridge.h @@ -1,6 +1,8 @@ #pragma once #include "parsers/parser.h" +#include "parsers/oram_parser.h" #include extern std::shared_ptr g_parser; +extern std::shared_ptr o_parser; diff --git a/toolkit/datahub/CMakeLists.txt b/toolkit/datahub/CMakeLists.txt index aec2a806..b59c9486 100644 --- a/toolkit/datahub/CMakeLists.txt +++ b/toolkit/datahub/CMakeLists.txt @@ -4,3 +4,17 @@ install(TARGETS data_provider DESTINATION "${bin_install_dir}") AddClangTidy(data_provider) EnableCoverage(data_provider) + +add_executable(oram_data_provider seal_oram_file.cpp ) +target_link_libraries(oram_data_provider stbox_common core core_stdeth core_gmssl) +install(TARGETS oram_data_provider + DESTINATION "${bin_install_dir}") +AddClangTidy(oram_data_provider) +EnableCoverage(oram_data_provider) + +add_executable(get_root_hash get_root_hash.cpp ) +target_link_libraries(get_root_hash core) +install(TARGETS get_root_hash + DESTINATION "${bin_install_dir}") +AddClangTidy(get_root_hash) +EnableCoverage(get_root_hash) \ No newline at end of file diff --git a/toolkit/datahub/get_root_hash.cpp b/toolkit/datahub/get_root_hash.cpp new file mode 100644 index 00000000..8b50e991 --- /dev/null +++ b/toolkit/datahub/get_root_hash.cpp @@ -0,0 +1,92 @@ +#include "ypc/core/oram_sealed_file.h" +#include "ypc/core/version.h" +#include "ypc/core/oram_sealed_file.h" + +#include +#include +#include +#include + +boost::program_options::variables_map parse_command_line(int argc, + char *argv[]) { + namespace bp = boost::program_options; + bp::options_description all("Read root hash options"); + bp::options_description general("General Options"); + bp::options_description get_root_hash_opts("Seal Data Options"); + + // clang-format off + get_root_hash_opts.add_options() + ("data-url", bp::value(), "Sealed Data URL") + ("output", bp::value(), "root hash file path"); + + general.add_options() + ("help", "help message") + ("version", "show version"); + + // clang-format on + + all.add(general).add(get_root_hash_opts); + + boost::program_options::variables_map vm; + boost::program_options::store( + boost::program_options::parse_command_line(argc, argv, all), vm); + + if (vm.count("help") != 0u) { + std::cout << all << std::endl; + exit(-1); + } + if (vm.count("version") != 0u) { + std::cout << ypc::get_ypc_version() << std::endl; + exit(-1); + } + return vm; +} + + +int main(int argc, char *argv[]) { + boost::program_options::variables_map vm; + + try { + vm = parse_command_line(argc, argv); + } catch (const std::exception &e) { + std::cerr << e.what() << std::endl; + std::cerr << "invalid cmd line parameters!" << std::endl; + return -1; + } + if (vm.count("data-url") == 0u) { + std::cerr << "data not specified!" << std::endl; + return -1; + } + if (vm.count("output") == 0u) { + std::cerr << "output not specified" << std::endl; + return -1; + } + + std::string data_file = vm["data-url"].as(); + std::string output = vm["output"].as(); + + auto sosf = std::make_shared(data_file); + + sosf->open_for_write(); + + ypc::memref me_hash; + bool ret = sosf->read_root_hash(me_hash); + ypc::bytes root_hash(me_hash.data(), me_hash.size()); + if(!ret) { + std::cout << "Cannot read root hash" << std::endl; + return -1; + } + + std::ofstream ofs; + ofs.open(output); + if (!ofs.is_open()) { + std::cout << "Cannot open file " << output << "\n"; + return -1; + } + + ofs << root_hash; + ofs.close(); + + std::cout << root_hash; + return 0; +} \ No newline at end of file diff --git a/toolkit/datahub/seal_oram_file.cpp b/toolkit/datahub/seal_oram_file.cpp new file mode 100644 index 00000000..c8472db9 --- /dev/null +++ b/toolkit/datahub/seal_oram_file.cpp @@ -0,0 +1,641 @@ +#include "ypc/common/crypto_prefix.h" +#include "ypc/common/limits.h" +#include "ypc/core/ntobject_file.h" +#include "ypc/core/privacy_data_reader.h" +#include "ypc/core/oram_sealed_file.h" +#include "ypc/core/version.h" +#include "ypc/corecommon/crypto/gmssl.h" +#include "ypc/corecommon/crypto/stdeth.h" +#include "ypc/corecommon/nt_cols.h" +#include "ypc/corecommon/oram_types.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using stx_status = stbox::stx_status; +using namespace ypc; +using ntt = ypc::nt; +using oram_ntt = ypc::oram::nt; +bool flag = false; + +define_nt(input_buf, std::string); +typedef ff::net::ntpackage<0, input_buf> input_buf_t; + +class crypto_base { +public: + virtual uint32_t encrypt_message_with_prefix(const ypc::bytes &public_key, + const ypc::bytes &data, + uint32_t prefix, + ypc::bytes &cipher) = 0; + virtual uint32_t hash_256(const ypc::bytes &msg, ypc::bytes &hash) = 0; +}; +using crypto_ptr_t = std::shared_ptr; +template class crypto_tool : public crypto_base { +public: + using crypto_t = Crypto; + virtual uint32_t encrypt_message_with_prefix(const ypc::bytes &public_key, + const ypc::bytes &data, + uint32_t prefix, + ypc::bytes &cipher) { + return crypto_t::encrypt_message_with_prefix(public_key, data, prefix, + cipher); + } + virtual uint32_t hash_256(const ypc::bytes &msg, ypc::bytes &hash) { + return crypto_t::hash_256(msg, hash); + } +}; + +ypc::bytes random_string(size_t len) { + std::string ret(len, '0'); + static std::default_random_engine generator; + static std::uniform_int_distribution distribution(int('a'), int('z')); + static auto rand = std::bind(distribution, generator); + + for (size_t i = 0; i < len; i++) { + ret[i] = rand(); + } + return ypc::bytes(ret.data(), ret.size()); +} + +void push_dummy_block(std::vector& bucket_array, ypc::bytes &data_hash, + uint8_t count, uint64_t item_num_each_batch, uint64_t item_size, + const crypto_ptr_t &crypto_ptr, const ypc::bytes &public_key) { + for(uint8_t i = 0; i < count; ++i) { + oram_ntt::block_t b_block; + + std::vector dummy_batch; + for(uint32_t j = 0; j < item_num_each_batch; ++j) { + ypc::bytes dummy_item = random_string(item_size); + dummy_batch.push_back(dummy_item); + ypc::bytes k_hash = data_hash + dummy_item; + crypto_ptr->hash_256(k_hash, data_hash); + } + + bytes encrypted_dummy_batch; + ypc::bytes dummy_batch_str = + ypc::make_bytes::for_package(dummy_batch); + + // encrypt dummy batch + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, dummy_batch_str, ypc::utc::crypto_prefix_arbitrary, encrypted_dummy_batch); + if (status != 0u) { + std::stringstream ss; + ss << "encrypt " + << " data fail: " << stbox::status_string(status); + LOG(ERROR) << ss.str(); + std::cerr << ss.str(); + exit(1); + } + + b_block.set(0, 0, 0, encrypted_dummy_batch); + bucket_array.push_back(b_block); + } +} + +uint32_t get_leaf_label(uint32_t bucket_index, uint8_t level_num_L) { + // leftmost leaf node + uint32_t leftmost_leaf_index = (1 << level_num_L) - 1; + if(bucket_index >= leftmost_leaf_index) { + return bucket_index - leftmost_leaf_index + 1; + } + + // randomly select a path to the leaf node + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(0, 1); + + if(dis(gen) == 0) { + return get_leaf_label(2 * bucket_index + 1, level_num_L); + } + return get_leaf_label(2 * bucket_index + 2, level_num_L); +} + +void push_real_block(std::vector& bucket_array, ypc::bytes &data_hash, + uint32_t& block_id_value, uint32_t bucket_index, + std::vector &position_map_array, uint8_t level_num_L, + std::vector &batch, uint32_t &batch_str_size, + uint64_t item_num_each_batch, uint64_t item_size, + const crypto_ptr_t &crypto_ptr, const ypc::bytes &public_key) { + oram_ntt::block_t b_block; + uint32_t valid_item_num = batch.size(); + for(uint32_t i = 0; i < item_num_each_batch - valid_item_num; ++i) { + ypc::bytes item = random_string(item_size); + batch.push_back(item); + } + + for(auto &item : batch) { + ypc::bytes k_hash = data_hash + item; + crypto_ptr->hash_256(k_hash, data_hash); + } + + bytes encrypted_batch; + ypc::bytes batch_str = + ypc::make_bytes::for_package(batch); + // encrypt batch + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, batch_str, ypc::utc::crypto_prefix_arbitrary, encrypted_batch); + if (status != 0u) { + std::stringstream ss; + ss << "encrypt " + << " data fail: " << stbox::status_string(status); + LOG(ERROR) << ss.str(); + std::cerr << ss.str(); + exit(1); + } + + if(batch_str_size != encrypted_batch.size()) { + batch_str_size = encrypted_batch.size(); + } + + uint32_t b_leaf_label = get_leaf_label(bucket_index, level_num_L); + position_map_array[block_id_value] = b_leaf_label; + b_block.set(block_id_value++, b_leaf_label, valid_item_num, encrypted_batch); + bucket_array.push_back(b_block); + +} + +uint32_t seal_oram_file(const crypto_ptr_t &crypto_ptr, const std::string &plugin, + const std::string &file, const std::string &oram_sealed_file_path, + const ypc::bytes &public_key) { + // Read origin file use sgx to seal file + privacy_data_reader reader(plugin, file); + // std::string k(file); + // k = k + std::string(sealer_path); + + // magic string here! + + bytes item_data = reader.read_item_data(); + if (item_data.size() > ypc::utc::max_item_size) { + std::cerr << "only support item size that smaller than " + << ypc::utc::max_item_size << " bytes!" << std::endl; + return 1; + } + uint64_t item_number = reader.get_item_number(); + + std::cout << "Reading " << item_number << " items ..." << std::endl; + boost::progress_display pd(item_number); + uint counter = 0; + size_t batch_size = 0; + uint64_t item_size = item_data.size(); + + // the number of batch + uint64_t batch_num = 0; + // item_num_array records the number of items each batch contains + std::vector item_num_array; + uint64_t item_num_each_batch = 0; + while (!item_data.empty() && counter < item_number) { + ++item_num_each_batch; + batch_size += item_data.size(); + if (batch_size >= ypc::utc::max_item_size) { + item_num_array.push_back(item_num_each_batch); + item_num_each_batch = 0; + batch_size = 0; + ++batch_num; + } + + item_data = reader.read_item_data(); + if (item_data.size() > ypc::utc::max_item_size) { + std::cerr << "only support item size that smaller than " + << ypc::utc::max_item_size << " bytes!" << std::endl; + return 1; + } + ++pd; + ++counter; + } + + if(item_num_each_batch != 0) { + item_num_array.push_back(item_num_each_batch); + item_num_each_batch = 0; + batch_size = 0; + ++batch_num; + } + + LOG(INFO) << "batch_num: " << batch_num; + + // create id map + LOG(INFO) << "create id_map"; + reader.reset_for_read(); + + counter = 0; + item_num_each_batch = 0; + + std::vector id_map_array; + uint32_t batch_id = 1; + std::string item_index_field = reader.get_item_index_field(); + input_buf_t item_index_field_pkg; + item_index_field_pkg.set(item_index_field); + + bytes item_index_field_bytes = make_bytes::for_package(item_index_field_pkg); + + while (!item_index_field.empty() && counter < item_number) { + bytes item_index_field_hash; + crypto_ptr->hash_256(item_index_field_bytes, item_index_field_hash); + + oram_ntt::id_map_pair k_v; + k_v.set(item_index_field_hash, batch_id); + id_map_array.push_back(k_v); + + ++item_num_each_batch; + if (item_num_each_batch >= item_num_array[batch_id-1]) { + item_num_each_batch = 0; + ++batch_id; + } + + item_index_field = reader.get_item_index_field(); + input_buf_t item_index_field_pkg; + item_index_field_pkg.set(item_index_field); + item_index_field_bytes = make_bytes::for_package(item_index_field_pkg); + + ++pd; + ++counter; + } + + oram_ntt::id_map_t id_map_pkg; + id_map_pkg.set(id_map_array); + bytes id_map_bytes = make_bytes::for_package(id_map_pkg); + + + // create header + LOG(INFO) << "create header"; + + ypc::oram::header osf_header{}; + osf_header.block_num = batch_num; + uint32_t real_bucket_num = ceil(static_cast(osf_header.block_num) / ypc::oram::BucketSizeZ); + osf_header.level_num_L = ceil(log2(real_bucket_num + 1)) - 1; + osf_header.bucket_num_N = (1 << (osf_header.level_num_L + 1)) - 1; + osf_header.id_map_filepos = sizeof(osf_header); + osf_header.oram_tree_filepos = osf_header.id_map_filepos + id_map_bytes.size(); + + + // write header, id map + LOG(INFO) << "write header and id map"; + std::fstream osf(oram_sealed_file_path, std::ios::out | std::ios::binary); + if(!osf.is_open()) { + throw std::runtime_error("Failed to create oram sealed file: " + oram_sealed_file_path); + } + + osf.seekp(0, osf.beg); + osf.write((char *)&osf_header, sizeof(osf_header)); + osf.write((char *)id_map_bytes.data(), id_map_bytes.size()); + + + // write ORAM tree + LOG(INFO) << "write ORAM tree"; + std::vector data_hash_array; + item_num_each_batch = item_num_array.front(); + + // from which bucket to start writing real blocks + uint8_t lastbucket_realblocknum = osf_header.block_num % oram::BucketSizeZ; + uint32_t bucket_index = 0; // bucket index in ORAM tree + uint32_t block_id_value = 1; // block_id_value <= osf_header.block_num + + // write buckets full of dummy blocks + LOG(INFO) << "write buckets full of dummy blocks"; + osf.seekp(osf_header.oram_tree_filepos, osf.beg); + for(uint32_t i = 0; i < osf_header.bucket_num_N - real_bucket_num; ++i) { + std::vector bucket_array; + ypc::bytes data_hash; + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + push_dummy_block(bucket_array, data_hash, oram::BucketSizeZ, + item_num_each_batch, item_size, crypto_ptr, public_key); + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = make_bytes::for_package(bucket_pkg); + + // secondary encryption on the serialized bucket + // in order to encrypt the mapping relationship between block_id and leaf_label + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status != 0u) { + std::stringstream ss; + ss << "encrypt " + << " data fail: " << stbox::status_string(status); + LOG(ERROR) << ss.str(); + std::cerr << ss.str(); + exit(1); + } + + osf.write((char *)encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + + data_hash_array.push_back(data_hash); + + ++bucket_index; + } + + reader.reset_for_read(); + uint64_t batch_index = 0; + std::vector batch; + std::vector position_map_array(osf_header.block_num + 1, 0); + + // write the bucket that contains both real and dummy blocks + LOG(INFO) << "write the bucket that contains both real and dummy blocks"; + if(lastbucket_realblocknum != 0) { + --real_bucket_num; + std::vector bucket_array; + ypc::bytes data_hash; + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + push_dummy_block(bucket_array, data_hash, oram::BucketSizeZ - lastbucket_realblocknum, item_num_each_batch, item_size, crypto_ptr, public_key); + for(int i = 0; i < lastbucket_realblocknum; ++i) { + batch.clear(); + for(int j = 0; j < item_num_array[batch_index]; ++j) { + item_data = reader.read_item_data(); + batch.push_back(item_data); + ++pd; + } + push_real_block(bucket_array, data_hash, block_id_value, bucket_index, position_map_array, osf_header.level_num_L, batch, osf_header.batch_str_size, item_num_each_batch, item_size, crypto_ptr, public_key); + ++batch_index; + } + + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = make_bytes::for_package(bucket_pkg); + + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status != 0u) { + std::stringstream ss; + ss << "encrypt " + << " data fail: " << stbox::status_string(status); + LOG(ERROR) << ss.str(); + std::cerr << ss.str(); + exit(1); + } + + if(osf_header.bucket_str_size != encrypted_bucket_bytes.size()) { + osf_header.bucket_str_size = encrypted_bucket_bytes.size(); + } + + osf.write((char *)encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + + data_hash_array.push_back(data_hash); + + ++bucket_index; + } + + // write buckets full of real blocks + LOG(INFO) << "write buckets full of real blocks"; + for(uint32_t k = 0; k < real_bucket_num; ++k) { + std::vector bucket_array; + ypc::bytes data_hash; + crypto_ptr->hash_256(bytes("Fidelius"), data_hash); + for(int i = 0; i < oram::BucketSizeZ; ++i) { + batch.clear(); + for(int j = 0; j < item_num_array[batch_index]; ++j) { + item_data = reader.read_item_data(); + batch.push_back(item_data); + ++pd; + } + push_real_block(bucket_array, data_hash, block_id_value, bucket_index, position_map_array, osf_header.level_num_L, batch, osf_header.batch_str_size, item_num_each_batch, item_size, crypto_ptr, public_key); + ++batch_index; + } + + oram_ntt::bucket_pkg_t bucket_pkg; + bucket_pkg.set(bucket_array); + bytes bucket_str = make_bytes::for_package(bucket_pkg); + + bytes encrypted_bucket_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, bucket_str, ypc::utc::crypto_prefix_arbitrary, encrypted_bucket_bytes); + if (status != 0u) { + std::stringstream ss; + ss << "encrypt " + << " data fail: " << stbox::status_string(status); + LOG(ERROR) << ss.str(); + std::cerr << ss.str(); + exit(1); + } + + if(osf_header.bucket_str_size != encrypted_bucket_bytes.size()) { + osf_header.bucket_str_size = encrypted_bucket_bytes.size(); + } + + osf.write((char *)encrypted_bucket_bytes.data(), encrypted_bucket_bytes.size()); + + data_hash_array.push_back(data_hash); + + ++bucket_index; + } + + // write position_map + osf_header.position_map_filepos = osf.tellp(); + oram_ntt::position_map_t position_map_pkg; + position_map_pkg.set(position_map_array); + bytes position_map_bytes = make_bytes::for_package(position_map_pkg); + ypc::bytes encrypted_position_map_bytes; + uint32_t status = crypto_ptr->encrypt_message_with_prefix( + public_key, position_map_bytes, ypc::utc::crypto_prefix_arbitrary, encrypted_position_map_bytes); + if (status != 0u) { + std::stringstream ss; + ss << "encrypt " + << " data fail: " << stbox::status_string(status); + LOG(ERROR) << ss.str(); + std::cerr << ss.str(); + exit(1); + } + + osf.seekp(osf_header.position_map_filepos, osf.beg); + osf.write((char *)encrypted_position_map_bytes.data(), encrypted_position_map_bytes.size()); + + + // write merkle tree + osf_header.merkle_tree_filepos = osf.tellp(); + + for(int i = (1 << osf_header.level_num_L) - 2; i >= 0; --i) { + ypc::bytes k_hash = data_hash_array[i] + data_hash_array[2*i + 1]; + crypto_ptr->hash_256(k_hash, data_hash_array[i]); + + k_hash = data_hash_array[i] + data_hash_array[2*i + 2]; + crypto_ptr->hash_256(k_hash, data_hash_array[i]); + } + + osf.seekp(osf_header.merkle_tree_filepos, osf.beg); + for(auto &data_hash : data_hash_array) { + osf.write((char *)data_hash.data(), data_hash.size()); + } + + + osf_header.stash_filepos = osf.tellp(); + + // update and write osf_header + osf.seekp(0, osf.beg); + osf.write((char *)&osf_header, sizeof(osf_header)); + + osf.close(); + + LOG(INFO) << "osf_header.block_num :" << osf_header.block_num; + LOG(INFO) << "osf_header.bucket_num_N :" << osf_header.bucket_num_N; + LOG(INFO) << "osf_header.level_num_L :" << osf_header.level_num_L; + LOG(INFO) << "osf_header.bucket_str_size :" << osf_header.bucket_str_size; + LOG(INFO) << "osf_header.batch_str_size :" << osf_header.batch_str_size; + LOG(INFO) << "osf_header.id_map_filepos :" << osf_header.id_map_filepos; + LOG(INFO) << "osf_header.oram_tree_filepos :" << osf_header.oram_tree_filepos; + LOG(INFO) << "osf_header.position_map_filepos :" << osf_header.position_map_filepos; + LOG(INFO) << "osf_header.merkle_tree_filepos :" << osf_header.merkle_tree_filepos; + LOG(INFO) << "osf_header.stash_filepos :" << osf_header.stash_filepos; + LOG(INFO) << "osf_header.stash_size :" << osf_header.stash_size; + LOG(INFO) << "osf_header.item_num_each_batch :" << osf_header.item_num_each_batch; + LOG(INFO) << "osf_header.item_size :" << osf_header.item_size; + + + return 0; +} + +boost::program_options::variables_map parse_command_line(int argc, + char *argv[]) { + namespace bp = boost::program_options; + bp::options_description all("YeeZ Privacy Data Hub options"); + bp::options_description general("General Options"); + bp::options_description seal_data_opts("Seal Data Options"); + + // clang-format off + seal_data_opts.add_options() + ("crypto", bp::value()->default_value("stdeth"), "choose the crypto, stdeth/gmssl") + ("data-url", bp::value(), "Data URL") + ("plugin-path", bp::value(), "shared library for reading data") + ("use-publickey-file", bp::value(), "public key file") + ("use-publickey-hex", bp::value(), "public key") + ("sealed-data-url", bp::value(), "Sealed data URL") + ("output", bp::value(), "output meta file path"); + + + general.add_options() + ("help", "help message") + ("version", "show version"); + + // clang-format on + + all.add(general).add(seal_data_opts); + + boost::program_options::variables_map vm; + boost::program_options::store( + boost::program_options::parse_command_line(argc, argv, all), vm); + + if (vm.count("help") != 0u) { + std::cout << all << std::endl; + exit(-1); + } + if (vm.count("version") != 0u) { + std::cout << ypc::get_ypc_version() << std::endl; + exit(-1); + } + return vm; +} + +int main(int argc, char *argv[]) { + boost::program_options::variables_map vm; + try { + vm = parse_command_line(argc, argv); + } catch (const std::exception &e) { + std::cerr << e.what() << std::endl; + std::cerr << "invalid cmd line parameters!" << std::endl; + return -1; + } + if (vm.count("data-url") == 0u) { + std::cerr << "data not specified!" << std::endl; + return -1; + } + if (vm.count("sealed-data-url") == 0u) { + std::cerr << "sealed data url not specified" << std::endl; + return -1; + } + if (vm.count("output") == 0u) { + std::cerr << "output not specified" << std::endl; + return -1; + } + if (vm.count("plugin-path") == 0u) { + std::cerr << "library not specified" << std::endl; + return -1; + } + if ((vm.count("use-publickey-hex") == 0u) && (vm.count("use-publickey-file") == 0u)) { + std::cerr << "missing public key, use 'use-publickey-file' or " + "'use-publickey-hex'" + << std::endl; + return -1; + } + if (vm.count("crypto") == 0u) { + std::cerr << "crypto not specified" << std::endl; + return -1; + } + + ypc::bytes public_key; + if (vm.count("use-publickey-hex") != 0u) { + public_key = ypc::hex_bytes(vm["use-publickey-hex"].as()) + .as(); + } else if (vm.count("use-publickey-file") != 0u) { + boost::property_tree::ptree pt; + boost::property_tree::json_parser::read_json( + vm["use-publickey-file"].as(), pt); + public_key = pt.get("public-key"); + } + + std::string plugin = vm["plugin-path"].as(); + std::string data_file = vm["data-url"].as(); + std::string output = vm["output"].as(); + std::string oram_sealed_data_file = vm["sealed-data-url"].as(); + std::string crypto = vm["crypto"].as(); + + std::ofstream ofs; + ofs.open(output); + if (!ofs.is_open()) { + std::cout << "Cannot open file " << output << "\n"; + return -1; + } + ofs.close(); + + crypto_ptr_t crypto_ptr; + if (crypto == "stdeth") { + crypto_ptr = std::make_shared>(); + } else if (crypto == "gmssl") { + crypto_ptr = std::make_shared>(); + } else { + throw std::runtime_error("Unsupperted crypto type!"); + } + + auto status = seal_oram_file(crypto_ptr, plugin, data_file, oram_sealed_data_file, + public_key); + if (status != 0u) { + return -1; + } + + ofs.open(output); + if (!ofs.is_open()) { + std::cout << "Cannot open file " << output << "\n"; + return -1; + } + ofs << "data_url" + << " = " << data_file << "\n"; + ofs << "sealed_data_url" + << " = " << oram_sealed_data_file << "\n"; + ofs << "public_key" + << " = " << public_key << "\n"; + + privacy_data_reader reader(plugin, data_file); + ofs << "item_num" + << " = " << reader.get_item_number() << "\n"; + + // sample and format are optional + bytes sample = reader.get_sample_data(); + if (!sample.empty()) { + ofs << "sample_data" + << " = " << sample << "\n"; + } + std::string format = reader.get_data_format(); + if (!format.empty()) { + ofs << " data_fromat" + << " = " << format << "\n"; + } + ofs.close(); + + std::cout << "done sealing" << std::endl; + return 0; +} diff --git a/toolkit/ydump/nouse_bridge.cpp b/toolkit/ydump/nouse_bridge.cpp index 2c4b9024..a76c2226 100644 --- a/toolkit/ydump/nouse_bridge.cpp +++ b/toolkit/ydump/nouse_bridge.cpp @@ -20,6 +20,46 @@ uint32_t km_end_session_ocall(uint32_t session_id); uint32_t next_data_batch(const uint8_t *data_hash, uint32_t hash_size, uint8_t **data, uint32_t *len); void free_data_batch(uint8_t *data); + + +uint32_t km_session_request_oram_ocall(sgx_dh_msg1_t *dh_msg1, uint32_t *session_id); +uint32_t km_exchange_report_oram_ocall(sgx_dh_msg2_t *dh_msg2, + sgx_dh_msg3_t *dh_msg3, uint32_t session_id); +uint32_t km_send_request_oram_ocall(uint32_t session_id, + secure_message_t *req_message, + size_t req_message_size, size_t max_payload_size, + secure_message_t *resp_message, + size_t resp_message_size); +uint32_t km_end_session_oram_ocall(uint32_t session_id); + +uint32_t download_oram_params_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_num, uint32_t *bucket_num_N, uint8_t *level_num_L, + uint32_t *bucket_str_size, uint32_t *batch_str_size); +uint32_t get_block_id_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_id, + const uint8_t *param_hash, uint32_t param_hash_size); +uint32_t download_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t ** position_map, uint32_t *len); +uint32_t update_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t * position_map, uint32_t len); +uint32_t download_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t ** encrpypted_path, uint32_t *len); +uint32_t download_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t ** stash, uint32_t *len); +uint32_t update_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t * stash, uint32_t len); +uint32_t upload_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t * encrpypted_path, uint32_t len); +uint32_t download_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t ** merkle_hash, uint32_t *len); +uint32_t update_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t * merkle_hash, uint32_t len); + +uint32_t write_convert_data_structure(int64_t filepos, uint8_t * convert_data_bytes, uint32_t len); + +uint32_t download_convert_params_ocall(uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size); + } @@ -45,3 +85,84 @@ uint32_t next_data_batch(const uint8_t *data_hash, uint32_t hash_size, return 0; } void free_data_batch(uint8_t *data) { return; } + + + +uint32_t km_session_request_oram_ocall(sgx_dh_msg1_t *dh_msg1, + uint32_t *session_id) { + return 0; +} +uint32_t km_exchange_report_oram_ocall(sgx_dh_msg2_t *dh_msg2, + sgx_dh_msg3_t *dh_msg3, uint32_t session_id) { + return 0; +} +uint32_t km_send_request_oram_ocall(uint32_t session_id, + secure_message_t *req_message, + size_t req_message_size, size_t max_payload_size, + secure_message_t *resp_message, + size_t resp_message_size) { + return 0; +} +uint32_t km_end_session_oram_ocall(uint32_t session_id) { return 0; } + +uint32_t download_oram_params_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_num, uint32_t *bucket_num_N, uint8_t *level_num_L, + uint32_t *bucket_str_size, uint32_t *batch_str_size) { + return 0; +} + +uint32_t get_block_id_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t *block_id, + const uint8_t *param_hash, uint32_t param_hash_size) { + return 0; +} + +uint32_t download_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t ** position_map, uint32_t *len) { + return 0; +} + +uint32_t update_position_map_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t * position_map, uint32_t len) { + return 0; +} + +uint32_t download_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t ** encrpypted_path, uint32_t *len) { + return 0; +} + +uint32_t download_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t ** stash, uint32_t *len) { + return 0; +} + +uint32_t update_stash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint8_t * stash, uint32_t len) { + return 0; +} + +uint32_t upload_path_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t * encrpypted_path, uint32_t len) { + return 0; +} + +uint32_t download_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t ** merkle_hash, uint32_t *len) { + return 0; +} + +uint32_t update_merkle_hash_OCALL(const uint8_t *data_hash, uint32_t hash_size, + uint32_t leaf, uint8_t * merkle_hash, uint32_t len) { + return 0; +} + + +uint32_t write_convert_data_structure(int64_t filepos, uint8_t * convert_data_bytes, uint32_t len) { + return 0; +} + +uint32_t download_convert_params_ocall(uint32_t *block_num, long int *oram_tree_filepos, + uint64_t *item_num_each_batch, uint64_t *item_size) { + return 0; +} \ No newline at end of file diff --git a/ypc/core/src/core/CMakeLists.txt b/ypc/core/src/core/CMakeLists.txt index 8863f546..c33a35a6 100644 --- a/ypc/core/src/core/CMakeLists.txt +++ b/ypc/core/src/core/CMakeLists.txt @@ -10,6 +10,7 @@ add_library(core SHARED db.cpp configuration.cpp sealed_file.cpp + oram_sealed_file.cpp command_executor.cpp filesystem.cpp privacy_data_reader.cpp diff --git a/ypc/core/src/core/oram_sealed_file.cpp b/ypc/core/src/core/oram_sealed_file.cpp new file mode 100644 index 00000000..cfffbeee --- /dev/null +++ b/ypc/core/src/core/oram_sealed_file.cpp @@ -0,0 +1,66 @@ +#include "ypc/core/oram_sealed_file.h" + +namespace ypc { +namespace internal { +oram_sealed_file_base::oram_sealed_file_base(const std::string &file_path) : m_file(file_path) {} + +} // namespace internal + +simple_oram_sealed_file::simple_oram_sealed_file(const std::string &file_path) + : oram_sealed_file_base(file_path) {} + +void simple_oram_sealed_file::reset() { m_file.reset(); } + +void simple_oram_sealed_file::open_for_write() { m_file.open_for_write(); } + +void simple_oram_sealed_file::open_for_read() { m_file.open_for_read(); } + +bool simple_oram_sealed_file::download_oram_params(uint32_t *block_num, uint32_t *bucket_num_N, + uint8_t *level_num_L, uint32_t *bucket_str_size, uint32_t *batch_str_size) { + + *block_num = m_file.get_block_num(); + *bucket_num_N = m_file.get_bucket_num(); + *level_num_L = m_file.get_level_num(); + *bucket_str_size = m_file.get_bucket_str_size(); + *batch_str_size = m_file.get_batch_str_size(); + + return true; +} + +bool simple_oram_sealed_file::get_block_id(const bytes &item_index_field_hash, uint32_t *block_id) { + return m_file.get_block_id(item_index_field_hash, block_id); +} + +bool simple_oram_sealed_file::download_position_map(memref &posmap) {return m_file.download_position_map(posmap); } + +bool simple_oram_sealed_file::update_position_map(const uint8_t * position_map, uint32_t len) { + return m_file.update_position_map(position_map, len); +} + +bool simple_oram_sealed_file::download_path(uint32_t leaf, memref &en_path) { + return m_file.download_path(leaf, en_path); +} + +bool simple_oram_sealed_file::upload_path(uint32_t leaf, const uint8_t * encrpypted_path, uint32_t len) { + return m_file.upload_path(leaf, encrpypted_path, len); +} + +bool simple_oram_sealed_file::download_stash(memref &st) { return m_file.download_stash(st); } + +bool simple_oram_sealed_file::update_stash(const uint8_t * stash, uint32_t len) { + return m_file.update_stash(stash, len); +} + +bool simple_oram_sealed_file::read_root_hash(memref &root_hash) { + return m_file.read_root_hash(root_hash); +} + +bool simple_oram_sealed_file::download_merkle_hash(uint32_t leaf, memref &merkle_hash) { + return m_file.download_merkle_hash(leaf, merkle_hash); +} + +bool simple_oram_sealed_file::update_merkle_hash(uint32_t leaf, const uint8_t * merkle_hash, uint32_t len) { + return m_file.update_merkle_hash(leaf, merkle_hash, len); +} + +} \ No newline at end of file diff --git a/ypc/core/src/core/privacy_data_reader.cpp b/ypc/core/src/core/privacy_data_reader.cpp index 84ea1fb8..90d64bf1 100644 --- a/ypc/core/src/core/privacy_data_reader.cpp +++ b/ypc/core/src/core/privacy_data_reader.cpp @@ -53,6 +53,8 @@ privacy_data_reader::privacy_data_reader(const std::string &plugin_path, get_opt_func_with_name("get_sample_data"); m_get_data_format = get_opt_func_with_name("get_data_format"); + m_get_item_index_field = + get_opt_func_with_name("get_item_index_field"); m_handle = m_create_item_reader(m_extra_param.c_str(), m_extra_param.size()); if (m_handle == nullptr) { @@ -112,4 +114,19 @@ bytes privacy_data_reader::read_item_data() { } return bytes(buf, len); } + +std::string privacy_data_reader::get_item_index_field() { + if (m_get_item_index_field == nullptr) { + return std::string(); + } + // We use static buf here to optimize memory usage. + char buf[::ypc::utc::max_item_size] = {0}; + + int len = ::ypc::utc::max_item_size; + auto status = m_get_item_index_field(m_handle, buf, &len); + if (status != 0) { + return std::string(); + } + return std::string(buf, len); +} } // namespace ypc diff --git a/ypc/core/src/core_t/analyzer/CMakeLists.txt b/ypc/core/src/core_t/analyzer/CMakeLists.txt index 8c7b17b8..ba571bb7 100644 --- a/ypc/core/src/core_t/analyzer/CMakeLists.txt +++ b/ypc/core/src/core_t/analyzer/CMakeLists.txt @@ -1,3 +1,5 @@ +set(ASM_FILE ${PROJECT_SOURCE_DIR}/ypc/core/src/core_t/analyzer/oblivious_functions.asm) + set(src analyzer_version.cpp internal/keymgr_session.cpp @@ -9,6 +11,20 @@ set(src ${PROJECT_SOURCE_DIR}/hpda/src/engine/functor.cpp ) + + +# TODO:oblivious_functions.o是需要作为enclave的可信库还是非可信库？ +add_custom_target(asm_target ALL COMMAND nasm -f elf64 ${ASM_FILE} -o ${PROJECT_SOURCE_DIR}/lib/oblivious_functions.o) + +# add_custom_target( +# clean_oblivious_functions +# COMMAND ${CMAKE_COMMAND} -E remove +# ${PROJECT_SOURCE_DIR}/lib/oblivious_functions.o +# ) +# add_dependencies(clean clean_oblivious_functions) + + + add_trusted_library(analyzer_t analyzer SRCS ${src} @@ -28,7 +44,7 @@ target_include_directories(analyzer_t PUBLIC "$" ) -target_link_libraries(analyzer_t PUBLIC stbox_channel_t stbox_crypto_t) +target_link_libraries(analyzer_t PUBLIC stbox_channel_t stbox_crypto_t ${PROJECT_SOURCE_DIR}/lib/oblivious_functions.o) install(TARGETS analyzer_t EXPORT mod_analyzer_t diff --git a/ypc/core/src/core_t/analyzer/internal/keymgr_session.cpp b/ypc/core/src/core_t/analyzer/internal/keymgr_session.cpp index 53ad2780..53fa8d33 100644 --- a/ypc/core/src/core_t/analyzer/internal/keymgr_session.cpp +++ b/ypc/core/src/core_t/analyzer/internal/keymgr_session.cpp @@ -32,6 +32,26 @@ uint32_t keymgr_session::init_keymgr_session() { LOG(INFO) << "done create keymgr session"; return t; } + +uint32_t keymgr_session::init_keymgr_session_oram() { + if (keymgr_var::m_keymgr_session) { + return stbox::stx_status::success; + } + + keymgr_var::m_keymgr_session.reset(new stbox::dh_session_initiator( + stbox::ocall_cast(km_session_request_oram_ocall), + stbox::ocall_cast(km_exchange_report_oram_ocall), + stbox::ocall_cast(km_send_request_oram_ocall), + stbox::ocall_cast(km_end_session_oram_ocall))); + keymgr_var::m_keymgr_session->set_verify_peer(km_verify_peer_enclave_trust); + + LOG(INFO) << "done init keymgr session"; + + auto t = keymgr_var::m_keymgr_session->create_session(); + LOG(INFO) << "done create keymgr session"; + return t; +} + uint32_t keymgr_session::close_keymgr_session() { return keymgr_var::m_keymgr_session->close_session(); } diff --git a/ypc/core/src/core_t/analyzer/oblivious_functions.asm b/ypc/core/src/core_t/analyzer/oblivious_functions.asm new file mode 100644 index 00000000..bc38ae5d --- /dev/null +++ b/ypc/core/src/core_t/analyzer/oblivious_functions.asm @@ -0,0 +1,147 @@ +BITS 64 + +section .text + global omove + global oset_value + global oset_flag + global oset_bytes + +omove: + ;command: + ;omove(i,&(array[i]),loc,leaf,newLabel) + ;Linux : rdi,rsi,rdx,rcx,r8,r9 + + ;If the callee wishes to use registers RBP, RBX, and R12-R15, + ;it must restore their original values before returning control to the caller. + ;All others must be saved by the caller if it wishes to preserve their values + + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + + mov ebx, dword [rcx] ; Move value in leaf to rbx + mov r13d, dword [rsi] ; r13d holds array[i] + + cmp edi, edx ; Compare i and loc + + ;if i == loc + cmovz ebx, dword [rsi] ; move value pointed by rdx to rbx (array[i] to rbx ) + cmovz r13d, r8d ; move newLabel to array[i] + + mov dword [rcx], ebx ; Push in leaflabel/prevleaf to leaf + mov dword [rsi], r13d ; Push in newLabel/array[i] to array[i] + + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + + ret + +oset_value: + ; oset_value(&dest, value, flag); + ; Linux : rdi,rsi,rdx,rcx,r8,r9 + ; Callee-saved : RBP, RBX, and R12–R15 + + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + + mov ebx, dword [rdi] + + cmp edx, 1 + + cmovz ebx, esi + + mov dword [rdi], ebx + + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + + ret + +oset_flag: + ; oset_flag(&dest_flag, src_flag, flag); + ; Linux : rdi,rsi,rdx,rcx,r8,r9 + ; Callee-saved : RBP, RBX, and R12–R15 + + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + + mov bl, byte [rdi] + + cmp dl, 1 + + cmovz bx, si + + mov byte [rdi], bl + + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + + ret + +oset_bytes: + ; oset_bytes(dest_bytes.data(), src_bytes.data(), bytes_size, flag) + ; Linux : rdi,rsi,rdx,rcx,r8,r9 + ; Callee-saved : RBP, RBX, and R12–R15 + + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + + + ; RCX will be lost for loop, store flag from rcx to rbp (1 byte , so bpl) + mov bpl, cl + + ; Oblivious evaluation of flag + cmp bpl, 1 + + mov ecx, edx + + loop_start: + cmp bpl, 1 + + mov r14b, byte [rdi] + mov r15b, byte [rsi] + + cmovz r14, r15 + + mov byte [rdi], r14b + + inc rdi + inc rsi + + loop loop_start + + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + + ret \ No newline at end of file