parallel_tronconneuse.hpp
Go to the documentation of this file.
/*********************************************************************/
// dar - disk archive - a backup/restoration program
// Copyright (C) 2002-2023 Denis Corbin
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
//
// to contact the author, see the AUTHOR file
/*********************************************************************/
#ifndef PARALLEL_TRONCONNEUSE_HPP
#define PARALLEL_TRONCONNEUSE_HPP
#include "../my_config.h"
#include <string>
#include "infinint.hpp"
#include "archive_version.hpp"
#include "crypto_segment.hpp"
#include "heap.hpp"
#include "crypto_module.hpp"
#include "proto_tronco.hpp"
#include <libthreadar/libthreadar.hpp>
namespace libdar
{
// those class are used by the parallel_tronconneuse class to wrap the different
// type of threads. They are defined just after the parallel_tronconneuse definition
class read_below;
class write_below;
class crypto_worker;
enum class tronco_flags { normal = 0, stop = 1, eof = 2, die = 3, data_error = 4, exception_below = 5, exception_worker = 6, exception_error = 7 };
//
// the parallel_tronconneuse class that orchestrate all that
//
//
class parallel_tronconneuse : public proto_tronco
{
public:
parallel_tronconneuse(U_I workers,
U_32 block_size,
generic_file & encrypted_side,
const archive_version & reading_ver,
std::unique_ptr<crypto_module> & ptr);
parallel_tronconneuse(const parallel_tronconneuse & ref) = delete;
parallel_tronconneuse(parallel_tronconneuse && ref) = default;
parallel_tronconneuse & operator = (const parallel_tronconneuse & ref) = delete;
parallel_tronconneuse & operator = (parallel_tronconneuse && ref) noexcept = default;
~parallel_tronconneuse() noexcept;
virtual bool skippable(skippability direction, const infinint & amount) override;
virtual bool skip(const infinint & pos) override;
virtual bool skip_to_eof() override;
virtual bool skip_relative(S_I x) override;
virtual bool truncatable(const infinint & pos) const override { return false; };
virtual infinint get_position() const override { if(is_terminated()) throw SRC_BUG; return current_position; };
virtual void write_end_of_file() override { if(is_terminated()) throw SRC_BUG; sync_write(); };
virtual void set_initial_shift(const infinint & x) override;
virtual void set_callback_trailing_clear_data(trailing_clear_data_callback call_back) override;
virtual U_32 get_clear_block_size() const override { return clear_block_size; };
private:
// inherited from generic_file
virtual void inherited_read_ahead(const infinint & amount) override;
virtual U_I inherited_read(char *a, U_I size) override;
virtual void inherited_write(const char *a, U_I size) override;
virtual void inherited_truncate(const infinint & pos) override { throw SRC_BUG; };
virtual void inherited_sync_write() override;
virtual void inherited_flush_read() override;
virtual void inherited_terminate() override;
const archive_version & get_reading_version() const { return reading_ver; };
// internal data structure
enum class thread_status { running, suspended, dead };
// the fields
U_I num_workers;
U_32 clear_block_size;
infinint current_position;
infinint initial_shift;
archive_version reading_ver;
std::unique_ptr<crypto_module> crypto;
infinint (*mycallback)(generic_file & below, const archive_version & reading_ver);
generic_file* encrypted;
// fields used to represent possible status of subthreads and communication channel (the pipe)
U_I ignore_stop_acks;
thread_status t_status;
// the following stores data from the ratelier_gather to be provided for read() operation
// the lus_data/lus_flags is what is extracted from the ratelier_gather, both constitute
// the feedback channel from sub-threads to provide order acks and normal data
std::deque<std::unique_ptr<crypto_segment> > lus_data;
std::deque<signed int> lus_flags;
bool lus_eof;
bool check_bytes_to_skip;
// the following stores data going to ratelier_scatter for the write() operation
std::unique_ptr<crypto_segment> tempo_write;
infinint block_num;
// the datastructures shared among threads
std::shared_ptr<libthreadar::ratelier_scatter<crypto_segment> > scatter;
std::shared_ptr<libthreadar::ratelier_gather<crypto_segment> > gather;
std::shared_ptr<libthreadar::barrier> waiter;
std::shared_ptr<heap<crypto_segment> > tas;
// the child threads
std::deque<std::unique_ptr<crypto_worker> > travailleur;
std::unique_ptr<read_below> crypto_reader;
std::unique_ptr<write_below> crypto_writer;
bool send_read_order(tronco_flags order, const infinint & for_offset = 0);
void send_write_order(tronco_flags order);
void go_read();
void read_refill();
tronco_flags purge_ratelier_from_next_order(infinint pos = 0);
bool purge_unack_stop_order(const infinint & pos = 0);
bool find_offset_in_lus_data(const infinint & pos);
void run_threads();
void stop_threads();
void join_workers_only();
void join_threads();
static U_I get_ratelier_size(U_I num_worker) { return num_worker + num_worker/2; };
static U_I get_heap_size(U_I num_worker);
};
//
// read_below subthread used by parallel_tronconneuse
// to dispatch chunk of encrypted data to the workers
//
class read_below: public libthreadar::thread
{
public:
read_below(const std::shared_ptr<libthreadar::ratelier_scatter<crypto_segment> > & to_workers,
const std::shared_ptr<libthreadar::barrier> & waiter,
U_I num_workers,
U_I clear_block_size,
generic_file* encrypted_side,
const std::shared_ptr<heap<crypto_segment> > xtas,
infinint init_shift):
workers(to_workers),
waiting(waiter),
num_w(num_workers),
clear_buf_size(clear_block_size),
encrypted(encrypted_side),
tas(xtas),
initial_shift(init_shift),
reof(false),
trailing_clear_data(nullptr)
{ flag = tronco_flags::normal; };
~read_below() { if(ptr) tas->put(move(ptr)); kill(); join(); };
void set_callback_trailing_clear_data(trailing_clear_data_callback call_back) { trailing_clear_data = call_back; };
// *** //
// *** the method above should not be used anymore once the thread is running *** //
// *** //
void set_initial_shift(const infinint & x) { initial_shift = x; };
void set_pos(const infinint & pos) { skip_to = pos; };
void set_flag(tronco_flags val) { flag = val; };
const infinint & get_clear_flow_start() const { return clear_flow_start; };
const infinint & get_pos_in_flow() const { return pos_in_flow; };
protected:
virtual void inherited_run() override;
private:
std::shared_ptr<libthreadar::ratelier_scatter <crypto_segment> > workers;
std::shared_ptr<libthreadar::barrier> waiting;
U_I num_w;
U_I clear_buf_size;
generic_file* encrypted;
archive_version version;
std::shared_ptr<heap<crypto_segment> > tas;
infinint initial_shift;
bool reof;
trailing_clear_data_callback trailing_clear_data;
std::unique_ptr<crypto_segment> ptr;
infinint index_num;
// initialized by inherited_run() / get_ready_for_new_offset()
infinint crypt_offset;
U_I encrypted_buf_size;
// fields accessible by both the caller and the read_below thread
infinint skip_to;
tronco_flags flag;
infinint clear_flow_start;
infinint pos_in_flow;
void work();
infinint get_ready_for_new_offset();
void send_flag_to_workers(tronco_flags theflag);
// same function as the tronconneuse::position_clear2crypt
void position_clear2crypt(const infinint & pos,
infinint & file_buf_start,
infinint & clear_buf_start,
infinint & pos_in_buf,
infinint & block_num);
};
//
// write_below subthread used by parallel_tronconneuse
// to gather and write down encrypted data work from workers
//
class write_below: public libthreadar::thread
{
public:
write_below(const std::shared_ptr<libthreadar::ratelier_gather<crypto_segment> > & from_workers,
const std::shared_ptr<libthreadar::barrier> & waiter,
U_I num_workers,
generic_file* encrypted_side,
const std::shared_ptr<heap<crypto_segment> > xtas):
workers(from_workers),
waiting(waiter),
num_w(num_workers),
cur_num_w(0),
encrypted(encrypted_side),
tas(xtas),
error(false),
error_block(0)
{ if(encrypted == nullptr) throw SRC_BUG; };
~write_below() { kill(); join(); };
bool exception_pending() const { return error; };
const infinint & get_error_block() const { return error_block; };
protected:
virtual void inherited_run() override;
private:
std::shared_ptr<libthreadar::ratelier_gather<crypto_segment> > workers;
std::shared_ptr<libthreadar::barrier> waiting;
U_I num_w;
U_I cur_num_w;
generic_file* encrypted;
std::shared_ptr<heap<crypto_segment> > tas;
bool error;
infinint error_block; // last crypto block before error
std::deque<std::unique_ptr<crypto_segment> >ones;
std::deque<signed int> flags;
void work();
};
//
// the crypto_worker threads performing ciphering/deciphering
// of many data blocks in parallel
//
class crypto_worker: public libthreadar::thread
{
public:
crypto_worker(std::shared_ptr<libthreadar::ratelier_scatter <crypto_segment> > & read_side,
std::shared_ptr<libthreadar::ratelier_gather <crypto_segment> > & write_side,
std::shared_ptr<libthreadar::barrier> waiter,
std::unique_ptr<crypto_module> && ptr,
bool encrypt):
reader(read_side),
writer(write_side),
waiting(waiter),
crypto(move(ptr)),
do_encrypt(encrypt),
abort(status::fine)
{ if(!reader || !writer || !waiting || !crypto) throw SRC_BUG; };
virtual ~crypto_worker() { kill(); join(); };
protected:
virtual void inherited_run() override;
private:
enum class status { fine, inform, sent };
std::shared_ptr<libthreadar::ratelier_scatter <crypto_segment> > & reader;
std::shared_ptr<libthreadar::ratelier_gather <crypto_segment> > & writer;
std::shared_ptr<libthreadar::barrier> waiting;
std::unique_ptr<crypto_module> crypto;
bool do_encrypt; // if false do decrypt
std::unique_ptr<crypto_segment> ptr;
unsigned int slot;
status abort;
void work();
};
} // end of namespace
#endif
archive_version.hpp
class archive_version that rules which archive format to follow
libdar::archive_version
class archive_version manages the version of the archive format
Definition: archive_version.hpp:47
libdar::generic_file
this is the interface class from which all other data transfer classes inherit
Definition: generic_file.hpp:77
libdar::generic_file::is_terminated
bool is_terminated() const
Definition: generic_file.hpp:327
libdar::generic_file::sync_write
void sync_write()
write any pending data
libdar::infinint
the arbitrary large positive integer class
Definition: real_infinint.hpp:62
libdar::parallel_tronconneuse
this is a partial implementation of the generic_file interface to cypher/decypher data block by block...
Definition: parallel_tronconneuse.hpp:89
libdar::parallel_tronconneuse::send_read_order
bool send_read_order(tronco_flags order, const infinint &for_offset=0)
send and order to subthreads and gather acks from them
libdar::parallel_tronconneuse::go_read
void go_read()
wake up threads in read mode when necessary
libdar::parallel_tronconneuse::inherited_terminate
virtual void inherited_terminate() override
this protected inherited method is now private for inherited classes of tronconneuse
libdar::parallel_tronconneuse::inherited_write
virtual void inherited_write(const char *a, U_I size) override
inherited from generic_file
libdar::parallel_tronconneuse::get_clear_block_size
virtual U_32 get_clear_block_size() const override
returns the block size given to constructor
Definition: parallel_tronconneuse.hpp:153
libdar::parallel_tronconneuse::run_threads
void run_threads()
reset the interthread datastructure and launch the threads
libdar::parallel_tronconneuse::inherited_sync_write
virtual void inherited_sync_write() override
this protected inherited method is now private for inherited classes of tronconneuse
libdar::parallel_tronconneuse::get_position
virtual infinint get_position() const override
inherited from generic_file
Definition: parallel_tronconneuse.hpp:132
libdar::parallel_tronconneuse::join_threads
void join_threads()
wait for threads to finish and eventually rethrow their exceptions in current thread
libdar::parallel_tronconneuse::write_end_of_file
virtual void write_end_of_file() override
in write_only mode indicate that end of file is reached
Definition: parallel_tronconneuse.hpp:140
libdar::parallel_tronconneuse::skippable
virtual bool skippable(skippability direction, const infinint &amount) override
inherited from generic_file
libdar::parallel_tronconneuse::purge_unack_stop_order
bool purge_unack_stop_order(const infinint &pos=0)
removing the ignore_stop_acks pending on the pipe
libdar::parallel_tronconneuse::set_callback_trailing_clear_data
virtual void set_callback_trailing_clear_data(trailing_clear_data_callback call_back) override
libdar::parallel_tronconneuse::parallel_tronconneuse
parallel_tronconneuse(parallel_tronconneuse &&ref)=default
move constructor
libdar::parallel_tronconneuse::crypto
std::unique_ptr< crypto_module > crypto
the crypto module use to cipher / uncipher block of data
Definition: parallel_tronconneuse.hpp:198
libdar::parallel_tronconneuse::purge_ratelier_from_next_order
tronco_flags purge_ratelier_from_next_order(infinint pos=0)
purge the ratelier from the next order which is provided as returned value
libdar::parallel_tronconneuse::skip_to_eof
virtual bool skip_to_eof() override
inherited from generic_file
libdar::parallel_tronconneuse::reading_ver
archive_version reading_ver
archive format we follow
Definition: parallel_tronconneuse.hpp:197
libdar::parallel_tronconneuse::parallel_tronconneuse
parallel_tronconneuse(U_I workers, U_32 block_size, generic_file &encrypted_side, const archive_version &reading_ver, std::unique_ptr< crypto_module > &ptr)
This is the constructor.
libdar::parallel_tronconneuse::skip
virtual bool skip(const infinint &pos) override
inherited from generic_file
libdar::parallel_tronconneuse::~parallel_tronconneuse
~parallel_tronconneuse() noexcept
destructor
libdar::parallel_tronconneuse::operator=
parallel_tronconneuse & operator=(const parallel_tronconneuse &ref)=delete
assignment operator
libdar::parallel_tronconneuse::skip_relative
virtual bool skip_relative(S_I x) override
inherited from generic_file
libdar::parallel_tronconneuse::inherited_truncate
virtual void inherited_truncate(const infinint &pos) override
this prorected inherited method is now private for inherited classed of tronconneuse
Definition: parallel_tronconneuse.hpp:174
libdar::parallel_tronconneuse::inherited_read
virtual U_I inherited_read(char *a, U_I size) override
this protected inherited method is now private for inherited classes of tronconneuse
libdar::parallel_tronconneuse::check_bytes_to_skip
bool check_bytes_to_skip
whether to check for bytes to skip
Definition: parallel_tronconneuse.hpp:215
libdar::parallel_tronconneuse::send_write_order
void send_write_order(tronco_flags order)
send order in write mode
libdar::parallel_tronconneuse::parallel_tronconneuse
parallel_tronconneuse(const parallel_tronconneuse &ref)=delete
copy constructor
libdar::parallel_tronconneuse::ignore_stop_acks
U_I ignore_stop_acks
how much stop ack still to be read (aborted stop order context)
Definition: parallel_tronconneuse.hpp:204
libdar::parallel_tronconneuse::initial_shift
infinint initial_shift
the offset in the "encrypted" below layer at which starts the encrypted data
Definition: parallel_tronconneuse.hpp:196
libdar::parallel_tronconneuse::clear_block_size
U_32 clear_block_size
size of a clear block
Definition: parallel_tronconneuse.hpp:194
libdar::parallel_tronconneuse::inherited_flush_read
virtual void inherited_flush_read() override
this protected inherited method is now private for inherited classes of tronconneuse
libdar::parallel_tronconneuse::truncatable
virtual bool truncatable(const infinint &pos) const override
inherited from generic_file
Definition: parallel_tronconneuse.hpp:130
libdar::parallel_tronconneuse::num_workers
U_I num_workers
number of worker threads
Definition: parallel_tronconneuse.hpp:193
libdar::parallel_tronconneuse::find_offset_in_lus_data
bool find_offset_in_lus_data(const infinint &pos)
flush lus_data/lus_flags up to requested pos offset to be found or all data has been removed
libdar::parallel_tronconneuse::stop_threads
void stop_threads()
end threads taking into account the fact they may be suspended on the barrier
libdar::parallel_tronconneuse::t_status
thread_status t_status
wehther child thread are waiting us on the barrier
Definition: parallel_tronconneuse.hpp:205
libdar::parallel_tronconneuse::read_refill
void read_refill()
fill lus_data/lus_flags from ratelier_gather if these are empty
libdar::parallel_tronconneuse::set_initial_shift
virtual void set_initial_shift(const infinint &x) override
this method to modify the initial shift. This overrides the constructor "no_initial_shift" of the con...
libdar::parallel_tronconneuse::inherited_read_ahead
virtual void inherited_read_ahead(const infinint &amount) override
this protected inherited method is now private for inherited classes of tronconneuse
libdar::parallel_tronconneuse::current_position
infinint current_position
current position for the upper layer perspective (modified by skip*, inherited_read/write,...
Definition: parallel_tronconneuse.hpp:195
libdar::parallel_tronconneuse::join_workers_only
void join_workers_only()
call by join_threads() below just code simplification around exception handling
crypto_module.hpp
per block cryptography implementation
crypto_segment.hpp
defines unit block of information ciphered as once
libdar::trailing_clear_data_callback
infinint(* trailing_clear_data_callback)(generic_file &below, const archive_version &reading_ver)
the trailing_clear_data_callback call back is a mean by which the upper layer cat tell when encrypted...
Definition: proto_tronco.hpp:58
libdar::tronco_flags
tronco_flags
status flags used between parallel_tronconneuse and its sub-threads
Definition: parallel_tronconneuse.hpp:64
heap.hpp
heap data structure (relying on FIFO)
infinint.hpp
switch module to limitint (32 ou 64 bits integers) or infinint
libdar
libdar namespace encapsulate all libdar symbols
Definition: archive.hpp:47
proto_tronco.hpp
defines common interface for tronconneuse and parallel_tronconneuse