parallel_tronconneuse.hpp

Go to the documentation of this file.

/*********************************************************************/
// dar - disk archive - a backup/restoration program
// Copyright (C) 2002-2024 Denis Corbin
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
//
// to contact the author, see the AUTHOR file
/*********************************************************************/



#ifndef PARALLEL_TRONCONNEUSE_HPP
#define PARALLEL_TRONCONNEUSE_HPP

#include "../my_config.h"
#include <string>

#include "infinint.hpp"
#include "archive_version.hpp"
#include "crypto_segment.hpp"
#include "heap.hpp"
#include "crypto_module.hpp"
#include "proto_tronco.hpp"

#include <libthreadar/libthreadar.hpp>

namespace libdar
{


        // those class are used by the parallel_tronconneuse class to wrap the different
        // type of threads. They are defined just after the parallel_tronconneuse definition
    class read_below;
    class write_below;
    class crypto_worker;


    enum class tronco_flags { normal = 0, stop = 1, eof = 2, die = 3, data_error = 4, exception_below = 5, exception_worker = 6, exception_error = 7 };


        //
        // the parallel_tronconneuse class that orchestrate all that
        //
        //




    class parallel_tronconneuse : public proto_tronco
    {
    public:

        parallel_tronconneuse(U_I workers,
                              U_32 block_size,
                              generic_file & encrypted_side,
                              const archive_version & reading_ver,
                              std::unique_ptr<crypto_module> & ptr);

        parallel_tronconneuse(const parallel_tronconneuse & ref) = delete;

        parallel_tronconneuse(parallel_tronconneuse && ref) = default;

        parallel_tronconneuse & operator = (const parallel_tronconneuse & ref) = delete;

        parallel_tronconneuse & operator = (parallel_tronconneuse && ref) noexcept = default;

        ~parallel_tronconneuse() noexcept;

        virtual bool skippable(skippability direction, const infinint & amount) override;
        virtual bool skip(const infinint & pos) override;
        virtual bool skip_to_eof() override;
        virtual bool skip_relative(S_I x) override;
        virtual bool truncatable(const infinint & pos) const override { return false; };
        virtual infinint get_position() const override { if(is_terminated()) throw SRC_BUG; return current_position; };


        virtual void write_end_of_file() override { if(is_terminated()) throw SRC_BUG; sync_write(); };



        virtual void set_initial_shift(const infinint & x) override;

        virtual void set_callback_trailing_clear_data(trailing_clear_data_callback call_back) override;

        virtual U_32 get_clear_block_size() const override { return clear_block_size; };

    private:

            // inherited from generic_file

        virtual void inherited_read_ahead(const infinint & amount) override;

        virtual U_I inherited_read(char *a, U_I size) override;


        virtual void inherited_write(const char *a, U_I size) override;


        virtual void inherited_truncate(const infinint & pos) override { throw SRC_BUG; };

        virtual void inherited_sync_write() override;


        virtual void inherited_flush_read() override;

        virtual void inherited_terminate() override;

        const archive_version & get_reading_version() const { return reading_ver; };

            // internal data structure
        enum class thread_status { running, suspended, dead };

            // the fields

        U_I num_workers;               
        U_32 clear_block_size;         
        infinint current_position;     
        infinint initial_shift;        
        archive_version reading_ver;   
        std::unique_ptr<crypto_module> crypto;  
        infinint (*mycallback)(generic_file & below, const archive_version & reading_ver);
        generic_file* encrypted;

            // fields used to represent possible status of subthreads and communication channel (the pipe)

        U_I ignore_stop_acks;          
        thread_status t_status;        


            // the following stores data from the ratelier_gather to be provided for read() operation
            // the lus_data/lus_flags is what is extracted from the ratelier_gather, both constitute
            // the feedback channel from sub-threads to provide order acks and normal data

        std::deque<std::unique_ptr<crypto_segment> > lus_data;
        std::deque<signed int> lus_flags;
        bool lus_eof;
        bool check_bytes_to_skip; 

            // the following stores data going to ratelier_scatter for the write() operation

        std::unique_ptr<crypto_segment> tempo_write;
        infinint block_num;

            // the datastructures shared among threads

        std::shared_ptr<libthreadar::ratelier_scatter<crypto_segment> > scatter;
        std::shared_ptr<libthreadar::ratelier_gather<crypto_segment> > gather;
        std::shared_ptr<libthreadar::barrier> waiter;
        std::shared_ptr<heap<crypto_segment> > tas;

            // the child threads

        std::deque<std::unique_ptr<crypto_worker> > travailleur;
        std::unique_ptr<read_below> crypto_reader;
        std::unique_ptr<write_below> crypto_writer;




        bool send_read_order(tronco_flags order, const infinint & for_offset = 0);

        void send_write_order(tronco_flags order);

        void go_read();

        void read_refill();


        tronco_flags purge_ratelier_from_next_order(infinint pos = 0);


        bool purge_unack_stop_order(const infinint & pos = 0);


        bool find_offset_in_lus_data(const infinint & pos);

        void run_threads();

        void stop_threads();

    void join_workers_only();

        void join_threads();


        static U_I get_ratelier_size(U_I num_worker) { return num_worker + num_worker/2; };
        static U_I get_heap_size(U_I num_worker);
    };


        //
        // read_below subthread used by parallel_tronconneuse
        // to dispatch chunk of encrypted data to the workers
        //

    class read_below: public libthreadar::thread
    {
    public:
        read_below(const std::shared_ptr<libthreadar::ratelier_scatter<crypto_segment> > & to_workers, 
                   const std::shared_ptr<libthreadar::barrier> & waiter, 
                   U_I num_workers,              
                   U_I clear_block_size,         
                   generic_file* encrypted_side, 
                   const std::shared_ptr<heap<crypto_segment> > xtas, 
                   infinint init_shift):         
            workers(to_workers),
            waiting(waiter),
            num_w(num_workers),
            clear_buf_size(clear_block_size),
            encrypted(encrypted_side),
            tas(xtas),
            initial_shift(init_shift),
            reof(false),
            trailing_clear_data(nullptr)
        { flag = tronco_flags::normal; };

        ~read_below() { if(ptr) tas->put(move(ptr)); kill(); join(); };

        void set_callback_trailing_clear_data(trailing_clear_data_callback call_back) { trailing_clear_data = call_back; };

            // *** //
            // *** the method above should not be used anymore once the thread is running *** //
            // *** //

        void set_initial_shift(const infinint & x) { initial_shift = x; };


        void set_pos(const infinint & pos) { skip_to = pos; };


        void set_flag(tronco_flags val) { flag = val; };


    const infinint & get_clear_flow_start() const { return clear_flow_start; };


    const infinint & get_pos_in_flow() const { return pos_in_flow; };


    protected:
    virtual void inherited_run() override;

    private:
    std::shared_ptr<libthreadar::ratelier_scatter <crypto_segment> > workers; 
    std::shared_ptr<libthreadar::barrier> waiting;                            
    U_I num_w;                                                                
    U_I clear_buf_size;                                                       
    generic_file* encrypted;                                                  
    archive_version version;                                                  
    std::shared_ptr<heap<crypto_segment> > tas;                               
    infinint initial_shift;                                                   
    bool reof;                                                                
    trailing_clear_data_callback trailing_clear_data;                         
    std::unique_ptr<crypto_segment> ptr;                                      
    infinint index_num;                                                       


        // initialized by inherited_run() / get_ready_for_new_offset()

    infinint crypt_offset;                                                    
    U_I encrypted_buf_size;                                                   

        // fields accessible by both the caller and the read_below thread

    infinint skip_to;          
    tronco_flags flag;         
    infinint clear_flow_start; 
    infinint pos_in_flow;      

    void work();
    infinint get_ready_for_new_offset();
    void send_flag_to_workers(tronco_flags theflag);

        // same function as the tronconneuse::position_clear2crypt
    void position_clear2crypt(const infinint & pos,
                  infinint & file_buf_start,
                  infinint & clear_buf_start,
                  infinint & pos_in_buf,
                  infinint & block_num);

    };


    //
    // write_below subthread used by parallel_tronconneuse
    // to gather and write down encrypted data work from workers
    //

    class write_below: public libthreadar::thread
    {
    public:
    write_below(const std::shared_ptr<libthreadar::ratelier_gather<crypto_segment> > & from_workers,
            const std::shared_ptr<libthreadar::barrier> & waiter,
            U_I num_workers,
            generic_file* encrypted_side,
            const std::shared_ptr<heap<crypto_segment> > xtas):
        workers(from_workers),
        waiting(waiter),
        num_w(num_workers),
        cur_num_w(0),
        encrypted(encrypted_side),
        tas(xtas),
        error(false),
        error_block(0)
    { if(encrypted == nullptr) throw SRC_BUG; };

    ~write_below() { kill(); join(); };

    bool exception_pending() const { return error; };
    const infinint & get_error_block() const { return error_block; };

    protected:
    virtual void inherited_run() override;

    private:
    std::shared_ptr<libthreadar::ratelier_gather<crypto_segment> > workers;
    std::shared_ptr<libthreadar::barrier> waiting;
    U_I num_w;
    U_I cur_num_w;
    generic_file* encrypted; 
    std::shared_ptr<heap<crypto_segment> > tas;
    bool error;
    infinint error_block; // last crypto block before error
    std::deque<std::unique_ptr<crypto_segment> >ones;
    std::deque<signed int> flags;

    void work();
    };


    //
    // the crypto_worker threads performing ciphering/deciphering
    // of many data blocks in parallel
    //


    class crypto_worker: public libthreadar::thread
    {
    public:
    crypto_worker(std::shared_ptr<libthreadar::ratelier_scatter <crypto_segment> > & read_side,
            std::shared_ptr<libthreadar::ratelier_gather <crypto_segment> > & write_side,
            std::shared_ptr<libthreadar::barrier> waiter,
            std::unique_ptr<crypto_module> && ptr,
            bool encrypt):
        reader(read_side),
        writer(write_side),
        waiting(waiter),
        crypto(move(ptr)),
        do_encrypt(encrypt),
        abort(status::fine)
    { if(!reader || !writer || !waiting || !crypto) throw SRC_BUG; };

    virtual ~crypto_worker() { kill(); join(); };

    protected:
    virtual void inherited_run() override;

    private:
    enum class status { fine, inform, sent };

    std::shared_ptr<libthreadar::ratelier_scatter <crypto_segment> > & reader;
    std::shared_ptr<libthreadar::ratelier_gather <crypto_segment> > & writer;
    std::shared_ptr<libthreadar::barrier> waiting;
    std::unique_ptr<crypto_module> crypto;
    bool do_encrypt; // if false do decrypt
    std::unique_ptr<crypto_segment> ptr;
    unsigned int slot;
    status abort;

    void work();
    };



} // end of namespace

#endif