marcelb/mySQL
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Integrate asynco in library, optimize maintenance on both engine

Browse Source
This commit is contained in:
mbandic 2024-09-27 10:57:45 +00:00
parent 4c50ba7416
commit c910c78633
6 changed files with 176 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.vscode/settings.json vendored
View File

@ -61,6 +61,8 @@
"cinttypes": "cpp",
"typeinfo": "cpp",
"any": "cpp",
"variant": "cpp"
"variant": "cpp",
"*.ipp": "cpp",
"bitset": "cpp"
}
}

55
README.md
View File

@ -13,7 +13,7 @@ A small framework for basic MySQL database operations via MySQL/Connector++
- Response object
- Thread safe
- Exceptions and log error callback
- Can use external time loop for connection management
- Support my Asynco wrapper around Boost ASIO and support threads
## Installation
@ -22,6 +22,11 @@ First install dependency MySQL/Connector++
```
sudo apt install libmysqlcppconn-dev
```
If you are going to use with an Asynco wrapper, download the archive from the profile and install the Boost dependencies
```
sudo apt install libboost-all-dev
```
Just download the latest release and unzip it into your project. You can turn it on with:
@ -29,7 +34,15 @@ Just download the latest release and unzip it into your project. You can turn it
#include "mysql/lib/mysql.hpp"
using namespace marcelb;
```
Compiling
```
# use with my Asynco lib
g++ -DMYSQL_USE_ASYNCO test.cpp ../src/* ../../asynco/src/* -o test.o -lmysqlcppconn -lpthread
# or use without asnyco (in multithread)
g++ test.cpp ../src/* ../../asynco/src/* -o test.o -lmysqlcppconn -lpthread
```
## Usage
### Internal engine
@ -46,6 +59,14 @@ using namespace marcelb::mysql;
*/
MySQL mydb("tcp://192.168.2.10:3306", "user_nm", "passss", "my_db", 5);
mydb.on_error = [](const string& error) {
cout << error << endl;
};
mydb.on_connect = []() {
cout << "Init all pool connection done" << endl;
};
/**
* Use ------------------
*/ | |
@ -72,34 +93,16 @@ try { | | | |
}
```
### External engine
### Run async with Asynco
As I developed quite a few wrappers that have some internal thread, I realized that it was inefficient and made it possible to call the necessary functions periodically outside (one thread per whole application or timer (ASIO), or my asynco wrapper).
```c++
#include "../lib/mysql.hpp"
using namespace marcelb::mysql;
#include "../../asynco/lib/timers.hpp"
using namespace marcelb::asynco;
/**
* Init
*/
MySQL mydb("tcp://192.168.2.10:3306", "user_nm", "passss", "my_db", 5, time_loop_type::external);
periodic mysql_maintenance ( [&mydb] () {
mydb.periodic_maintenance();
}, MYSQL_PERIODIC_INTERNAL_TIME);
mydb.set_on_error( [](const string& error) {
cout << error << endl; // print or log
});
/**
* You can call multiple queries asynchronously
*/
auto a1 = atask ( [&mydb] () {
auto a1 = async_ ( [&mydb] () {
try {
auto response = mydb.exec<int,string>("SELECT id,domain FROM records WHERE enabled = 1;");
for (auto row : response) {
@ -110,7 +113,7 @@ auto a1 = atask ( [&mydb] () {
}
});
auto a2 = atask ( [&mydb] () {
auto a2 = async_ ( [&mydb] () {
try {
auto response = mydb.exec<string,string>("SELECT zonename,auth_key FROM zones;");
for (auto row : response) {
@ -121,7 +124,7 @@ auto a2 = atask ( [&mydb] () {
}
});
auto a3 = atask ( [&mydb] () {
auto a3 = async_ ( [&mydb] () {
try {
auto response = mydb.exec<string,string>("SELECT username,email FROM users WHERE enabled = 1;");
for (auto row : response) {
@ -132,9 +135,9 @@ auto a3 = atask ( [&mydb] () {
}
});
wait(a1);
wait(a2);
wait(a3);
await(a1);
await(a2);
await(a3);
```
## License

66
lib/mysql.hpp
View File

@ -8,7 +8,6 @@
#include <string>
#include <vector>
#include <tuple>
#include <ctime>
#include <chrono>
using namespace std;
@ -39,6 +38,8 @@ namespace mysql {
*
*/
#define MYSQL_PERIODIC_INTERNAL_TIME 5000
#define MYSQL_MAX_CONNECTION_INIT_SAME_TIME 10
#define MYSQL_DELAYED_CONNECT_TIME 1000
/**
* A class for creating sql responses
@ -99,13 +100,16 @@ class MySQL {
queue<Connection*> connection_pool;
string path, username, password, database;
uint32_t pool_size;
bool run_tloop = true;
const uint32_t max_t = thread::hardware_concurrency();
uint32_t t = 1;
#ifdef MYSQL_USE_ASYNCO
periodic p_loop;
// delayed d_connect;
#else
future<void> tloop_future;
future<void> tloop_future, connect_f;
bool run_tloop = true;
#endif
time_t last_loop_time;
/**
* Open one database server connection
@ -154,8 +158,57 @@ class MySQL {
void _tloop(uint32_t b, uint32_t e);
/**
*
*/
vector<pair<int, int>> divide_and_conquer(int n, int t) {
vector<pair<int, int>> pairs;
int part_size = n / t;
int residue = n % t; // Ostatak koji treba rasporediti
for (int i = 0; i < t; i++) {
int start = i * part_size + std::min(i, residue);
int end = start + part_size + (i < residue ? 1 : 0);
pairs.push_back(make_pair(start, end));
}
return pairs;
}
/**
* Automatically adjusts
*/
void auto_adjusts(const uint32_t duration) {
if (duration > (MYSQL_PERIODIC_INTERNAL_TIME*2)/3) {
if (t < max_t) {
t++; // povećaj broj asinkronih zadataka drugi put
}
else if (on_error) {
on_error("Periodic maintenance takes too long");
}
}
else if (duration < MYSQL_PERIODIC_INTERNAL_TIME/3 && t > 1) {
t--; // smanji nema potrebe
}
}
#ifndef MYSQL_USE_ASYNCO
/**
*
*/
uint64_t timestamp() {
return chrono::duration_cast<chrono::milliseconds>(
chrono::system_clock::now().time_since_epoch()
).count();
};
#endif
public:
function<void(const string&)> on_error;
function<void()> on_connect;
uint32_t connect_trys = 3;
@ -206,13 +259,12 @@ public:
release_connection(connection);
} catch (sql::SQLException& e) {
throw runtime_error(e.what());
// std::cerr << "SQLState: " << e.getSQLState() << std::endl;
// std::cerr << "Error code: " << e.getErrorCode() << std::endl;
// cerr << "SQLState: " << e.getSQLState() << endl;
// cerr << "Error code: " << e.getErrorCode() << endl;
}
return result;
}
/**
* Destruktor
* close all connections

107
src/mysql.cpp
View File

@ -3,24 +3,55 @@
marcelb::mysql::MySQL::MySQL(const string _path, const string _username, const string _password, const string _db, const uint32_t _available):
#ifdef MYSQL_USE_ASYNCO
p_loop(periodic( [&] () {
cout << "U asynco" << endl;
const uint64_t start = rtime_ms();
auto pairs = divide_and_conquer(connection_pool.size(), t);
vector<future<void>> fuve;
for (auto& pair : pairs) {
fuve.push_back(async_ ([&, pair](){
_tloop(pair.first, pair.second);
}));
}
for (auto& fu : fuve) {
try {
auto start = rtime_ms();
_tloop(0, connection_pool.size());
cout << "loop--------------------------- nema error, trajalo: " << rtime_ms() - start << endl;
await_(fu);
} catch (...) {
// cout << "Bude neki error u loopu" << endl;
if(on_error) {
on_error("Bude neki error u loopu");
on_error("Error in maintenance periodic loop.");
}
}
}
auto_adjusts(rtime_ms() - start);
}, MYSQL_PERIODIC_INTERNAL_TIME)),
#else
tloop_future (async(launch::async, [&](){
while (run_tloop) {
cout << "U STD async" << endl;
usleep(MYSQL_PERIODIC_INTERNAL_TIME*1000);
_tloop(0, connection_pool.size());
const uint64_t start = timestamp();
auto pairs = divide_and_conquer(connection_pool.size(), t);
vector<future<void>> fuve;
for (auto& pair : pairs) {
fuve.push_back(async (launch::async, [&, pair](){
_tloop(pair.first, pair.second);
}));
}
for (auto& fu : fuve) {
try {
fu.get();
} catch (...) {
if(on_error) {
on_error("Error in maintenance periodic loop.");
}
}
}
auto_adjusts(timestamp() - start);
}
return;
})),
@ -33,9 +64,6 @@ marcelb::mysql::MySQL::MySQL(const string _path, const string _username, const s
drv = get_mysql_driver_instance();
connect_pool();
// set on initialization to avoid the error
last_loop_time = time(nullptr);
}
@ -68,11 +96,28 @@ Connection* marcelb::mysql::MySQL::create_connection() {
}
void marcelb::mysql::MySQL::connect_pool() {
lock_guard<mutex> lock(io);
auto connect_ = [&]() -> void {
const uint32_t tens = pool_size/MYSQL_MAX_CONNECTION_INIT_SAME_TIME;
for (uint32_t i=0; i<pool_size; i++) {
Connection* connection = create_connection();
{
lock_guard<mutex> lock(io);
connection_pool.push(connection);
}
if ((i+1)%tens == 0) {
usleep(MYSQL_DELAYED_CONNECT_TIME*1000);
}
}
if (on_connect) {
on_connect();
}
};
#ifdef MYSQL_USE_ASYNCO
async_ (connect_);
#else
connect_f = async (launch::async, connect_);
#endif
}
void marcelb::mysql::MySQL::disconnect_pool() {
@ -108,35 +153,18 @@ bool marcelb::mysql::MySQL::disconnect_connection(Connection* connection) {
}
void marcelb::mysql::MySQL::_tloop(uint32_t b, uint32_t e) {
if (!run_tloop) {
return;
}
for (size_t i=b; i<connection_pool.size() && i<e; i++) {
try {
Connection *conn = nullptr;
{
lock_guard<mutex> lock(io);
conn = connection_pool.front();
connection_pool.pop();
}
Connection *conn = nullptr; // provjeri ovdje svugdje možeš li koristiti release i ocupacy sada
conn = occupy_connection();
if (conn->isValid()) {
cout << "Validno----" << endl;
connection_pool.push(conn);
condition.notify_one();
release_connection(conn);
} else {
cout << "Nije validno----" << endl;
if (!conn->isClosed()){
cout << "Zatvori----" << endl;
conn->close();
disconnect_connection(conn);
}
Connection *n_conn = create_connection();
{
cout << "Otvori----" << endl;
lock_guard<mutex> lock(io);
connection_pool.push(n_conn);
condition.notify_one();
}
release_connection(n_conn);
}
} catch (const SQLException &error) {
@ -145,16 +173,9 @@ void marcelb::mysql::MySQL::_tloop(uint32_t b, uint32_t e) {
}
}
}
last_loop_time = time(nullptr);
}
Connection* marcelb::mysql::MySQL::occupy_connection() {
if (last_loop_time + (MYSQL_PERIODIC_INTERNAL_TIME*3/1000) < time(nullptr)) {
if (on_error) {
on_error("The time loop is not executing properly");
}
}
unique_lock<mutex> lock(io);
while (connection_pool.empty()) {
condition.wait(lock);
@ -171,8 +192,10 @@ void marcelb::mysql::MySQL::release_connection(Connection* connection) {
}
marcelb::mysql::MySQL::~MySQL() {
#ifdef MYSQL_USE_ASYNCO
p_loop.stop();
#else
run_tloop = false;
#ifndef MYSQL_USE_ASYNCO
tloop_future.get();
#endif
disconnect_pool();

4
test/compile.sh
View File

@ -1,2 +1,2 @@
# g++ -DMYSQL_USE_ASYNCO test.cpp ../src/* ../../asynco/src/* -o test.o -lmysqlcppconn -lpthread
g++ test.cpp ../src/* ../../asynco/src/* -o test.o -lmysqlcppconn -lpthread
g++ -DMYSQL_USE_ASYNCO test.cpp ../src/* ../../asynco/src/* -o test.o -lmysqlcppconn -lpthread
# g++ test.cpp ../src/* ../../asynco/src/* -o test.o -lmysqlcppconn -lpthread

12
test/test.cpp
View File

@ -15,17 +15,21 @@ using namespace marcelb::mysql;
int main() {
auto inis = rtime_ms();
try {
const int n = 5;
const int n = 10;
// MySQL mydb("tcp://192.168.2.10:3306", "dinio", "H€r5elfInd1aH@nds", "dinio", 5, time_loop_type::internal);
MySQL mydb("tcp://bitelex.ddns.net:3306", "dinio", "H€r5elfInd1aH@nds", "dinio", n);
// MySQL mydb("tcp://bitelex.ddns.net:3306", "dinio", "H€r5elfInd1aH@nds", "dinio", 5);
cout << "init: " << rtime_ms() - inis << endl;
cout << "--------------init: " << rtime_ms() - inis << endl;
mydb.on_error = [](const string& error) {
cout << error << endl;
};
mydb.on_connect = []() {
cout << "Init all pool connection done" << endl;
};
// periodic mysql_tloop ( [&mydb] () {
// auto l_start = rtime_ms();
// vector<future<void>> to_wait;
@ -203,7 +207,9 @@ while (true) {
} catch (const SQLException error) {
cout << error.what() << endl;
} catch (const string error) {
} catch (const exception& error) {
cout << error.what() << endl;
} catch (const string& error) {
cout << error << endl;
} catch (...) {
cout << "Jebi ga" << endl;