From 47b9e082bf43912b47ec0f81a62b36b1a3d8e339 Mon Sep 17 00:00:00 2001 From: SalomeSep <78850039+ApusDT@users.noreply.github.com> Date: Tue, 5 Oct 2021 16:27:28 +0200 Subject: [PATCH 1/9] Add files via upload --- omnn/math/Cacheefdb.h | 130 ++++++++++++++++++++++++++ omnn/math/Cachefdb.cpp | 203 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 333 insertions(+) create mode 100644 omnn/math/Cacheefdb.h create mode 100644 omnn/math/Cachefdb.cpp diff --git a/omnn/math/Cacheefdb.h b/omnn/math/Cacheefdb.h new file mode 100644 index 0000000000..971319b85c --- /dev/null +++ b/omnn/math/Cacheefdb.h @@ -0,0 +1,130 @@ +#pragma once +//pgrama says include the file once + + +#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE +//check the following function is defined + + +#include +//copy and paste files into other files + +#else +//namespace creates a block where all functions inside of it are asociated to the namespace +namespace fdb_cli{ +class DB; +} +#endif +//if OPENMIND.. is defined then erases the else part, otherwise it keeps it + + +//#include +#include +#include +#include + +#include +//definition and implementation are in the same file, .hpp calls a class + +#include "Variable.h" + +namespace omnn::math { + +namespace fs = boost::filesystem; // permite referenciar (acceder) a un tipo, objeto, función o enumerador incluso si su identificador está oculto +//define la primera gran clase + class Cache + { + public: //public declares a public member of a class. It allows access to a member inside and outside the class + using path_str_t = fs::path; -//define la variable path_srt_t como path de la clase fs class fs and method path (?) función path dentro de clase fs + // :: = permite referenciar (acceder) a un tipo, objeto, función o enumerador incluso si su identificador está oculto + using CheckCacheResult = std::pair; + + template + + class CachedValueBase : public std::future { + using base = std::future; + ResultT result = {}; + bool extracted = {}; + void Extract() { //void is a function that doesn't return any value + if(!extracted){ + extracted = true; + result = base::get(); //class base method get + } + } + protected: //once it's gotten the value becomes protected i.e. only member of class' functions can access to it + //but it cannot be accesed from other classes + ResultT& Get() { + Extract(); + return result; + } + public: + using base::base; + + CachedValueBase(base&& b) : base(std::move(b)) + {} + + operator bool(){ + return (extracted || + ( base::valid() + && base::wait_for(std::chrono::seconds(0)) == std::future_status::ready + )) && Get().first + ; + } + + bool NotInCache(){ + return (extracted || + ( base::valid() + && base::wait_for(std::chrono::seconds(0)) == std::future_status::ready + )) && !Get().first; + } + }; + + class Cached : public CachedValueBase { + using base = CachedValueBase; + public: + using base::base; + using base::operator bool; +// Cached(base&& b); + operator Valuable(); + }; + + using val_set_t = Valuable::solutions_t; + using CheckCachedSet = std::pair; + + class CachedSet : public CachedValueBase { + using base = CachedValueBase; + public: + using base::base; + using base::operator bool; +// CachedSet(base&& b); + operator val_set_t(); + }; + + private: + leveldb::DB* db = nullptr; + path_str_t path; + + void DbOpen(); + + protected: + std::string Value(const std::string& key); + + public: + Cache(const path_str_t& path); + ~Cache(); + + Cached AsyncFetch(const Valuable& v, bool itIsOptimized = false); + CheckCacheResult GetOne(const std::string& key, const Valuable::va_names_t& vaNames, bool itIsOptimized = false); + + CachedSet AsyncFetchSet(const Valuable& v, bool itIsOptimized = false); + CheckCachedSet GetSet(const std::string& key, const Valuable::va_names_t& vaNames, bool itIsOptimized = false); + + bool Set(const std::string& key, const std::string& v); + void AsyncSet(std::string&& key, std::string&& v); + void AsyncSetSet(const Valuable& key, const val_set_t& v); + }; +} + +#define CACHE(name) \ + auto cachename##name = std::string(#name) + ".solutions"; \ + Cache name(cachename##name); diff --git a/omnn/math/Cachefdb.cpp b/omnn/math/Cachefdb.cpp new file mode 100644 index 0000000000..cac0c54ed1 --- /dev/null +++ b/omnn/math/Cachefdb.cpp @@ -0,0 +1,203 @@ +// +//Created by Sergej Krivonos on 31.12.2019.// +#include "Cachefdb.h" + +#include +#include +#include +#include +#include + +#include + +using namespace omnn::math; + +namespace { + +constexpr size_t MaxThreadsForCacheStoring = 1024; +using CacheStoringTask = std::future; +using CacheStoringTasksQueue = std::deque; + +void CleanupReadyTasks(CacheStoringTasksQueue &CacheStoringTasks) { + while (CacheStoringTasks.size() && + ((CacheStoringTasks.front().valid() && + CacheStoringTasks.front().wait_for(std::chrono::seconds(0)) == + std::future_status::ready) || + CacheStoringTasks.size() >= MaxThreadsForCacheStoring)) { + if (!CacheStoringTasks.front().get()) + std::cerr << "Cache storing task failed" << std::endl; + CacheStoringTasks.pop_front(); + } +} + +#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE +void DeleteDB(const Cache::path_str_t& path) { + if(fs::exists(path)) + fs::remove_all(path); +} +#endif +} + +void Cache::DbOpen() { +#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE +# if OPENMIND_MATH_CACHE_VOLATILE + DeleteDB(path); +# endif + foundationdb::Options options; + options.create_if_missing = true; + options.compression = foundationdb::kSnappyCompression; + options.max_file_size = 512*1024*1024; + options.block_size = 16*1024; + options.paranoid_checks=true; + + foundationdb::Status status; + auto strPath = path.string(); + while (!((status = foundationdb::DB::Open(options, strPath, &db)).ok())) { + auto err = strPath + " DB connection error"; + std::cerr << err << status.ToString() << std::endl; + if(status.IsIOError()){ + std::cout << "DB is busy. Waiting. Retrying in 5 sec." << std::endl; + using namespace std::chrono_literals; + std::this_thread::sleep_for(5s); + } else { +#ifndef NDEBUG + DeleteDB(path); + if (!foundationdb::DB::Open(options, strPath, &db).ok()) +#endif + throw err; + } + } +#endif +} + +Cache::Cache(const Cache::path_str_t &path) { + this->path = path; + DbOpen(); +} + +omnn::math::Cache::~Cache() { +#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE + if(db) + delete db; // hopely it has overloaded operator delete inside its image +#endif +} + +Cache::Cached Cache::AsyncFetch(const Valuable &v, bool itIsOptimized) { + using self_t = std::remove_reference::type; + auto&& task = std::async(std::launch::async, &self_t::GetOne, + this, v.str(), v.VaNames(), itIsOptimized); +// Cache::Cached cached(task); + return std::move(task); +} + +Cache::CheckCacheResult Cache::GetOne(const std::string &key, + const Valuable::va_names_t &vaNames, + bool itIsOptimized) { + CheckCacheResult cachedValue; +#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE + std::string value; + cachedValue.first = db->Get(foundationdb::ReadOptions(), key, &value).ok(); + if(cachedValue.first){ + cachedValue.second = Valuable(value, vaNames, itIsOptimized); +#ifndef NDEBUG +// std::cout << "fetched from cache " << key << " => " << cachedValue.second << std::endl; +#endif + assert(cachedValue.second.is_optimized() == itIsOptimized); +#ifdef ALLOW_CACHE_UPGRADE + auto gotValStr = cachedValue.second.str(); + if (gotValStr != value) { + #ifndef NDEBUG + std::cout << "upgrading cached value for key " << key << ": " << value << " => " << gotValStr << std::endl; + #endif + AsyncSet(std::string(key), std::move(gotValStr)); + } +#endif + } +#endif + return cachedValue; +} + +Cache::CachedSet Cache::AsyncFetchSet(const Valuable& v, bool itIsOptimized){ + using self_t = std::remove_reference::type; + auto&& task = std::async(std::launch::async, &self_t::GetSet, + this, v.str(), v.VaNames(), itIsOptimized); + return std::move(task); +} + +namespace { +constexpr auto CachedSetDelimeters = "<|>"; +} +Cache::CheckCachedSet Cache::GetSet(const std::string& key, const Valuable::va_names_t& vaNames, bool itIsOptimized){ + CheckCachedSet cachedSet; +#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE + std::string value; + cachedSet.first = db->Get(leveldb::ReadOptions(), key, &value).ok(); + if(cachedSet.first){ + char* token = strtok(const_cast(value.c_str()), CachedSetDelimeters); + while(token){ + cachedSet.second.insert(Valuable(std::string_view(token), vaNames, itIsOptimized)); + token = strtok(nullptr, CachedSetDelimeters); + } +#ifndef NDEBUG +// std::cout << "fetched from cache " << key << " => " << value << std::endl; +#endif + } +#endif + return cachedSet; +} + +bool Cache::Set(const std::string &key, const std::string &v) { + return db +#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE + && db->Put(leveldb::WriteOptions(), key, v).ok() +#endif + ; +} + +void Cache::AsyncSet(std::string &&key, std::string &&v) { +#ifdef OPENMIND_MATH_USE_LEVELDB_CACHE + static CacheStoringTasksQueue CacheStoringTasks; + + CleanupReadyTasks(CacheStoringTasks); + + // add new task + CacheStoringTasks.emplace_back(std::async(std::launch::async, + [ This=this, // TODO: This=shared_from_this(), + k=std::move(key), v=std::move(v) + ](){ + return This->Set(k,v); + } + )); +#endif +} + +void Cache::AsyncSetSet(const Valuable& key, const val_set_t& v) { + std::stringstream ss; + ss << v; + auto s = ss.str(); + auto l = s.length(); + if (s[l-1] == ' ') + s[l-1] = 0; + AsyncSet(key.str(), std::move(s)); +} + +Cache::Cached::operator Valuable() { + assert(operator bool()); + auto& got = Get(); +#ifndef NDEBUG + assert(got.first); +// std::cout << "Used from cache: " << got.second << std::endl; +#endif + assert(got.second.is_optimized()); // if cached value is not optimized then just remove this assert + return got.second; +} + +Cache::CachedSet::operator Cache::val_set_t() { + assert(operator bool()); + auto got = Get(); + assert(got.first); +#ifndef NDEBUG +//std::cout << "Used from cache: [ " << got.second << "]" << std::endl;// +#endif + return got.second; +} From 0c8cdbe879a3c3072d1f8e3188365ff10292f5de Mon Sep 17 00:00:00 2001 From: SalomeSep <78850039+ApusDT@users.noreply.github.com> Date: Wed, 6 Oct 2021 21:25:19 +0200 Subject: [PATCH 2/9] Update Cachefdb.cpp --- omnn/math/Cachefdb.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/omnn/math/Cachefdb.cpp b/omnn/math/Cachefdb.cpp index cac0c54ed1..e65539e718 100644 --- a/omnn/math/Cachefdb.cpp +++ b/omnn/math/Cachefdb.cpp @@ -1,5 +1,4 @@ -// -//Created by Sergej Krivonos on 31.12.2019.// + #include "Cachefdb.h" #include From 90fe286eba7f29b97e718c91d076105f291cea1d Mon Sep 17 00:00:00 2001 From: SalomeSep <78850039+ApusDT@users.noreply.github.com> Date: Thu, 7 Oct 2021 09:26:37 +0200 Subject: [PATCH 3/9] Add files via upload --- omnn/math/CMakeLists.txt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/omnn/math/CMakeLists.txt b/omnn/math/CMakeLists.txt index e87d70db08..5776c5f2fb 100644 --- a/omnn/math/CMakeLists.txt +++ b/omnn/math/CMakeLists.txt @@ -1,3 +1,5 @@ +project(math) +cmake_minimum_required(VERSION 3.20) option(OPENMIND_MATH_USE_LEVELDB_CACHE "Use Google LevelDB library to cache solutions" TRUE) if(OPENMIND_MATH_USE_LEVELDB_CACHE) @@ -28,18 +30,18 @@ if(NOT WIN32) set(DEPENDENCIES ${DEPENDENCIES} boost_thread boost_serialization boost_system) endif() -lib(${DEPENDENCIES}) +link_libraries(${DEPENDENCIES}) -if (OPENMIND_USE_OPENCL) - target_include_directories(${this_target} PUBLIC - ${OpenCL_INCLUDE_DIR} - ) -endif() +if (OPENMIND_USE_OPENCL) + target_include_directories(${this_target} PUBLIC + ${OpenCL_INCLUDE_DIR} + ) +endif() -if (OPENMIND_BUILD_TESTS) +if (OPENMIND_BUILD_TESTS) add_subdirectory(test) endif () -if (OPENMIND_BUILD_3RD_PARTY_TESTS) +if (OPENMIND_BUILD_3RD_PARTY_TESTS) add_subdirectory(3rdPartyTests) endif () From 8e9140385d8f64760dc6ede707b0b9bb6ea0bb29 Mon Sep 17 00:00:00 2001 From: SalomeSep <78850039+ApusDT@users.noreply.github.com> Date: Thu, 7 Oct 2021 18:32:14 +0200 Subject: [PATCH 4/9] Add files via upload --- omnn/math/CachefdbTRY2.h | 110 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 omnn/math/CachefdbTRY2.h diff --git a/omnn/math/CachefdbTRY2.h b/omnn/math/CachefdbTRY2.h new file mode 100644 index 0000000000..8fed4d1a43 --- /dev/null +++ b/omnn/math/CachefdbTRY2.h @@ -0,0 +1,110 @@ +#pragma once +//pragma says include the file once + + +#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE +//check the following function is defined + + +#include + +#include +//copy and paste files into other files + +#else + +namespace fdb_cli{ +class FDB; +} +#endif + + + +//#include +#include +#include +#include + +#include +//definition and implementation are in the same file, .hpp calls a class + +#include "Variable.h" + +namespace omnn::math { + +namespace fs = boost::filesystem; + + class Cachefdb + { + public: + using path_str_t_fdb = fs::path; + + using CheckCache_fdb_Result = std::pair; + + template + + class Cached_fdb_ValueBase : public std::future { + using base_fdb = std::future; + ResultT_fdb result_fdb = {}; + bool extracted_fdb = {}; + void Extract_fdb() { + if(!extracted_fdb){ + extracted_fdb = true; + result_fdb = base::get(); //class base method get + } + } + protected: //once it's gotten the value becomes protected i.e. only member of class' functions can access to it + //but it cannot be accesed from other classes + ResultT_fdb& Get() { + Extract_fdb(); + return result_fdb; + } + public: + using base::base; + + Cached_fdb_ValueBase(base&& b) : base(std::move(b)) + {} + + operator bool(){ + return (extracted_fdb || + ( base::valid_fdb() + && base::wait_for(std::chrono::seconds(0)) == std::future_status::ready + )) && Get().first + ; + } + + bool NotInCache_fdb(){ + return (extracted_fdb || + ( base::valid_fdb() + && base::wait_for(std::chrono::seconds(0)) == std::future_status::ready + )) && !Get().first; + } + }; + + class Cached_fdb : public Cached_fdb_ValueBase { + using base = Cached_fdb_ValueBase; + public: + using base::base; + using base::operator bool; +// Cached_fdb(base&& b); + operator Valuable_fdb(); + }; + + using val_set_t_fdb = Valuable::solutions_t; + using CheckCached_fdb_Set = std::pair; + + class Cached_fdb_Set : public Cached_fdb_ValueBase { + using base = Cached_fdb_ValueBase; + public: + using base::base; + using base::operator bool; +// CachedSet_fdb(base&& b); + operator val_set_t(); + }; + + + + + + + \ No newline at end of file From 3beb7aa188d7f26c57f20c6c58d9190c258cc9dd Mon Sep 17 00:00:00 2001 From: SalomeSep <78850039+ApusDT@users.noreply.github.com> Date: Thu, 7 Oct 2021 18:39:08 +0200 Subject: [PATCH 5/9] Delete Cachefdb.cpp --- omnn/math/Cachefdb.cpp | 202 ----------------------------------------- 1 file changed, 202 deletions(-) delete mode 100644 omnn/math/Cachefdb.cpp diff --git a/omnn/math/Cachefdb.cpp b/omnn/math/Cachefdb.cpp deleted file mode 100644 index e65539e718..0000000000 --- a/omnn/math/Cachefdb.cpp +++ /dev/null @@ -1,202 +0,0 @@ - -#include "Cachefdb.h" - -#include -#include -#include -#include -#include - -#include - -using namespace omnn::math; - -namespace { - -constexpr size_t MaxThreadsForCacheStoring = 1024; -using CacheStoringTask = std::future; -using CacheStoringTasksQueue = std::deque; - -void CleanupReadyTasks(CacheStoringTasksQueue &CacheStoringTasks) { - while (CacheStoringTasks.size() && - ((CacheStoringTasks.front().valid() && - CacheStoringTasks.front().wait_for(std::chrono::seconds(0)) == - std::future_status::ready) || - CacheStoringTasks.size() >= MaxThreadsForCacheStoring)) { - if (!CacheStoringTasks.front().get()) - std::cerr << "Cache storing task failed" << std::endl; - CacheStoringTasks.pop_front(); - } -} - -#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE -void DeleteDB(const Cache::path_str_t& path) { - if(fs::exists(path)) - fs::remove_all(path); -} -#endif -} - -void Cache::DbOpen() { -#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE -# if OPENMIND_MATH_CACHE_VOLATILE - DeleteDB(path); -# endif - foundationdb::Options options; - options.create_if_missing = true; - options.compression = foundationdb::kSnappyCompression; - options.max_file_size = 512*1024*1024; - options.block_size = 16*1024; - options.paranoid_checks=true; - - foundationdb::Status status; - auto strPath = path.string(); - while (!((status = foundationdb::DB::Open(options, strPath, &db)).ok())) { - auto err = strPath + " DB connection error"; - std::cerr << err << status.ToString() << std::endl; - if(status.IsIOError()){ - std::cout << "DB is busy. Waiting. Retrying in 5 sec." << std::endl; - using namespace std::chrono_literals; - std::this_thread::sleep_for(5s); - } else { -#ifndef NDEBUG - DeleteDB(path); - if (!foundationdb::DB::Open(options, strPath, &db).ok()) -#endif - throw err; - } - } -#endif -} - -Cache::Cache(const Cache::path_str_t &path) { - this->path = path; - DbOpen(); -} - -omnn::math::Cache::~Cache() { -#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE - if(db) - delete db; // hopely it has overloaded operator delete inside its image -#endif -} - -Cache::Cached Cache::AsyncFetch(const Valuable &v, bool itIsOptimized) { - using self_t = std::remove_reference::type; - auto&& task = std::async(std::launch::async, &self_t::GetOne, - this, v.str(), v.VaNames(), itIsOptimized); -// Cache::Cached cached(task); - return std::move(task); -} - -Cache::CheckCacheResult Cache::GetOne(const std::string &key, - const Valuable::va_names_t &vaNames, - bool itIsOptimized) { - CheckCacheResult cachedValue; -#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE - std::string value; - cachedValue.first = db->Get(foundationdb::ReadOptions(), key, &value).ok(); - if(cachedValue.first){ - cachedValue.second = Valuable(value, vaNames, itIsOptimized); -#ifndef NDEBUG -// std::cout << "fetched from cache " << key << " => " << cachedValue.second << std::endl; -#endif - assert(cachedValue.second.is_optimized() == itIsOptimized); -#ifdef ALLOW_CACHE_UPGRADE - auto gotValStr = cachedValue.second.str(); - if (gotValStr != value) { - #ifndef NDEBUG - std::cout << "upgrading cached value for key " << key << ": " << value << " => " << gotValStr << std::endl; - #endif - AsyncSet(std::string(key), std::move(gotValStr)); - } -#endif - } -#endif - return cachedValue; -} - -Cache::CachedSet Cache::AsyncFetchSet(const Valuable& v, bool itIsOptimized){ - using self_t = std::remove_reference::type; - auto&& task = std::async(std::launch::async, &self_t::GetSet, - this, v.str(), v.VaNames(), itIsOptimized); - return std::move(task); -} - -namespace { -constexpr auto CachedSetDelimeters = "<|>"; -} -Cache::CheckCachedSet Cache::GetSet(const std::string& key, const Valuable::va_names_t& vaNames, bool itIsOptimized){ - CheckCachedSet cachedSet; -#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE - std::string value; - cachedSet.first = db->Get(leveldb::ReadOptions(), key, &value).ok(); - if(cachedSet.first){ - char* token = strtok(const_cast(value.c_str()), CachedSetDelimeters); - while(token){ - cachedSet.second.insert(Valuable(std::string_view(token), vaNames, itIsOptimized)); - token = strtok(nullptr, CachedSetDelimeters); - } -#ifndef NDEBUG -// std::cout << "fetched from cache " << key << " => " << value << std::endl; -#endif - } -#endif - return cachedSet; -} - -bool Cache::Set(const std::string &key, const std::string &v) { - return db -#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE - && db->Put(leveldb::WriteOptions(), key, v).ok() -#endif - ; -} - -void Cache::AsyncSet(std::string &&key, std::string &&v) { -#ifdef OPENMIND_MATH_USE_LEVELDB_CACHE - static CacheStoringTasksQueue CacheStoringTasks; - - CleanupReadyTasks(CacheStoringTasks); - - // add new task - CacheStoringTasks.emplace_back(std::async(std::launch::async, - [ This=this, // TODO: This=shared_from_this(), - k=std::move(key), v=std::move(v) - ](){ - return This->Set(k,v); - } - )); -#endif -} - -void Cache::AsyncSetSet(const Valuable& key, const val_set_t& v) { - std::stringstream ss; - ss << v; - auto s = ss.str(); - auto l = s.length(); - if (s[l-1] == ' ') - s[l-1] = 0; - AsyncSet(key.str(), std::move(s)); -} - -Cache::Cached::operator Valuable() { - assert(operator bool()); - auto& got = Get(); -#ifndef NDEBUG - assert(got.first); -// std::cout << "Used from cache: " << got.second << std::endl; -#endif - assert(got.second.is_optimized()); // if cached value is not optimized then just remove this assert - return got.second; -} - -Cache::CachedSet::operator Cache::val_set_t() { - assert(operator bool()); - auto got = Get(); - assert(got.first); -#ifndef NDEBUG -//std::cout << "Used from cache: [ " << got.second << "]" << std::endl;// -#endif - return got.second; -} From aedd341456167a780dbe8d51bb015b7977a88806 Mon Sep 17 00:00:00 2001 From: SalomeSep <78850039+ApusDT@users.noreply.github.com> Date: Thu, 7 Oct 2021 18:39:17 +0200 Subject: [PATCH 6/9] Delete Cacheefdb.h --- omnn/math/Cacheefdb.h | 130 ------------------------------------------ 1 file changed, 130 deletions(-) delete mode 100644 omnn/math/Cacheefdb.h diff --git a/omnn/math/Cacheefdb.h b/omnn/math/Cacheefdb.h deleted file mode 100644 index 971319b85c..0000000000 --- a/omnn/math/Cacheefdb.h +++ /dev/null @@ -1,130 +0,0 @@ -#pragma once -//pgrama says include the file once - - -#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE -//check the following function is defined - - -#include -//copy and paste files into other files - -#else -//namespace creates a block where all functions inside of it are asociated to the namespace -namespace fdb_cli{ -class DB; -} -#endif -//if OPENMIND.. is defined then erases the else part, otherwise it keeps it - - -//#include -#include -#include -#include - -#include -//definition and implementation are in the same file, .hpp calls a class - -#include "Variable.h" - -namespace omnn::math { - -namespace fs = boost::filesystem; // permite referenciar (acceder) a un tipo, objeto, función o enumerador incluso si su identificador está oculto -//define la primera gran clase - class Cache - { - public: //public declares a public member of a class. It allows access to a member inside and outside the class - using path_str_t = fs::path; -//define la variable path_srt_t como path de la clase fs class fs and method path (?) función path dentro de clase fs - // :: = permite referenciar (acceder) a un tipo, objeto, función o enumerador incluso si su identificador está oculto - using CheckCacheResult = std::pair; - - template - - class CachedValueBase : public std::future { - using base = std::future; - ResultT result = {}; - bool extracted = {}; - void Extract() { //void is a function that doesn't return any value - if(!extracted){ - extracted = true; - result = base::get(); //class base method get - } - } - protected: //once it's gotten the value becomes protected i.e. only member of class' functions can access to it - //but it cannot be accesed from other classes - ResultT& Get() { - Extract(); - return result; - } - public: - using base::base; - - CachedValueBase(base&& b) : base(std::move(b)) - {} - - operator bool(){ - return (extracted || - ( base::valid() - && base::wait_for(std::chrono::seconds(0)) == std::future_status::ready - )) && Get().first - ; - } - - bool NotInCache(){ - return (extracted || - ( base::valid() - && base::wait_for(std::chrono::seconds(0)) == std::future_status::ready - )) && !Get().first; - } - }; - - class Cached : public CachedValueBase { - using base = CachedValueBase; - public: - using base::base; - using base::operator bool; -// Cached(base&& b); - operator Valuable(); - }; - - using val_set_t = Valuable::solutions_t; - using CheckCachedSet = std::pair; - - class CachedSet : public CachedValueBase { - using base = CachedValueBase; - public: - using base::base; - using base::operator bool; -// CachedSet(base&& b); - operator val_set_t(); - }; - - private: - leveldb::DB* db = nullptr; - path_str_t path; - - void DbOpen(); - - protected: - std::string Value(const std::string& key); - - public: - Cache(const path_str_t& path); - ~Cache(); - - Cached AsyncFetch(const Valuable& v, bool itIsOptimized = false); - CheckCacheResult GetOne(const std::string& key, const Valuable::va_names_t& vaNames, bool itIsOptimized = false); - - CachedSet AsyncFetchSet(const Valuable& v, bool itIsOptimized = false); - CheckCachedSet GetSet(const std::string& key, const Valuable::va_names_t& vaNames, bool itIsOptimized = false); - - bool Set(const std::string& key, const std::string& v); - void AsyncSet(std::string&& key, std::string&& v); - void AsyncSetSet(const Valuable& key, const val_set_t& v); - }; -} - -#define CACHE(name) \ - auto cachename##name = std::string(#name) + ".solutions"; \ - Cache name(cachename##name); From f61719d065224c675b91c5604983da6b6bb342f4 Mon Sep 17 00:00:00 2001 From: SalomeSep <78850039+ApusDT@users.noreply.github.com> Date: Thu, 7 Oct 2021 18:43:21 +0200 Subject: [PATCH 7/9] Add files via upload --- omnn/math/CachefdbTRY2.cpp | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 omnn/math/CachefdbTRY2.cpp diff --git a/omnn/math/CachefdbTRY2.cpp b/omnn/math/CachefdbTRY2.cpp new file mode 100644 index 0000000000..af395fd6b8 --- /dev/null +++ b/omnn/math/CachefdbTRY2.cpp @@ -0,0 +1,3 @@ + +#include "CachefdbTRY2.h" + From 93e7b747fa3f03b2a3f1191911d80a575f9d343c Mon Sep 17 00:00:00 2001 From: SalomeSep <78850039+ApusDT@users.noreply.github.com> Date: Thu, 14 Oct 2021 09:18:32 +0200 Subject: [PATCH 8/9] Delete CachefdbTRY2.cpp --- omnn/math/CachefdbTRY2.cpp | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 omnn/math/CachefdbTRY2.cpp diff --git a/omnn/math/CachefdbTRY2.cpp b/omnn/math/CachefdbTRY2.cpp deleted file mode 100644 index af395fd6b8..0000000000 --- a/omnn/math/CachefdbTRY2.cpp +++ /dev/null @@ -1,3 +0,0 @@ - -#include "CachefdbTRY2.h" - From 9d106248c79a37d19e0da894acbecd1493d4240f Mon Sep 17 00:00:00 2001 From: SalomeSep <78850039+ApusDT@users.noreply.github.com> Date: Thu, 14 Oct 2021 09:22:18 +0200 Subject: [PATCH 9/9] Add files via upload --- omnn/math/CacheFDBtry3.h | 146 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 omnn/math/CacheFDBtry3.h diff --git a/omnn/math/CacheFDBtry3.h b/omnn/math/CacheFDBtry3.h new file mode 100644 index 0000000000..c3479dedff --- /dev/null +++ b/omnn/math/CacheFDBtry3.h @@ -0,0 +1,146 @@ +#pragma once + + + +#ifdef OPENMIND_MATH_USE_FOUNDATIONDB_CACHE + +#include + +#include + +#else + +namespace fdb_cli{ + class FDB; +} +#endif + + +//#include +//#include +#include +#include + +#include +#include "Variable.h" + +namespace omnn::math { + +namespace bi = boost::intrusive; + +class EvictablePage + { + public: + using path_str_t = bi::path + using CheckCacheResult = std::pair; + + template < typename pageCache> + + class Reference : public EvictablePageCache ::future { + using base = std::future;//creo que lo de abajo es lo mismo + virtual bool evict() = 0; // evic=extracted + + + //EvictablePage(Reference pageCache) : pageCache(pageCache) {} + // virtual ~EvictablePage(); + } + + + protected: + pageCache& Get() { + evict()//Extract(); + return pageCache; + } + + + public: + using bi::bi; + + Reference (bi&& b) : bi(std::move(b)) + {} + + operator bool(){ + return (evict || + ( bi::valid() + && bi::wait_for(std::chrono::seconds(0)) == std::future_status::ready + )) && Get().first + ; + } + + bool NotInCache(){ + return (evict || + ( bi::valid() + && bi::wait_for(std::chrono::seconds(0)) == std::future_status::ready + )) && !Get().first; + } + }; + +class EvictablePageCache : public ReferenceCounted + { + using List = + bi::list, &EvictablePage::path>>; + + public: + using bi::bi; + using bi::operator bool; +// EvictablePageCache (bi&& b); + operator Valuable(); + }; + + using val_set_t = Valuable::solutions_t; + using CheckCachedSet = std::pair; + + + + + + + class CachedSet : public ReferenceCounted { + using base = ReferenceCounted ; + public: + using bi::bi; + using bi::operator bool; +// CachedSet(bi&& b); + operator val_set_t(); + }; + + private: + fdb_cli::FDB* fdb_c = nullptr; + path_str_t path; + + + + // Opens a file that uses the FDB in-memory page cache + static Future> open(std::string filename, int flags, int mode) { + //TraceEvent("AsyncFileCachedOpen").detail("Filename", filename); + auto itr = openFiles.find(filename); + if (itr == openFiles.end()) { + auto f = open_impl(filename, flags, mode); + if (f.isReady() && f.isError()) + return f; + + auto result = openFiles.try_emplace(filename, f); + + // This should be inserting a new entry + ASSERT(result.second); + itr = result.first; + + // We return here instead of falling through to the outer scope so that we don't delete all references to + // the underlying file before returning + return itr->second.get(); + } + return itr->second.get(); + } + + + void DbOpen(); + protected: + std::string Value(const std::string& key); + + public: + Cachefdb(const path_str_t& path); + ~Cachefdb(); + + public: + Cachefdb(const path_str_t& path); + ~Cachefdb();