GCC 11.1.0 Module und ein Fehler



  • Problembeschreibung:
    Ich hatte vor einigen Tagen bereits den NUMA-Allocator-Simulator gepostet. Nun wollte ich damit ausprobieren, wie aufwendig es ist ein Projekt zu „modularisieren“ und erhalte nun eine etwas sonderbare Fehlermeldung, die natürlich in der Header Version nicht auftritt.

    Zuerst der Code in modularisierter Form

    /*
     *
     * NUMA_Allocator_Simulator.cc
     *
     *
     */
    
    export module NUMA_Allocator_Simulator;
    
    import <algorithm>;
    import <iostream>;
    import <memory>;
    import <iterator>;
    import <vector>;
    import <cstddef>;
    import <new>;
    
    #include <boost/core/noncopyable.hpp>
    
    /// Chunks are used to store all informations for each allocation.
    export struct Chunk {
    	size_t	size_;
    	void*	p_;
    
    	/// @param [in] s Size of memory [chunk](@ref Chunk).
    	/// @param [in] pointer The pointer to the memory [chunk](@ref Chunk).
    	Chunk (const size_t s, void* const pointer) : size_(s), p_(pointer) {}
    };
    
    /// @brief This function compares [Chunks](@ref Chunk) by their stored pointer address.
    /// @param [in] ck1 Chunk
    /// @param [in] ck2 Chunk
    /// @return ck1.p_ < ck2.p_
    export bool operator< (const Chunk& ck1, const Chunk& ck2) {
    	return (ck1.p_ < ck2.p_);
    }
    
    export struct ChunkIndex {
    	size_t	size_;
    	size_t	index_;
    };
    
    /// @brief This function compares [ChunkIndexes](@ref ChunkIndex) by their stored sizes.
    /// @param [in] ci1 ChunkIndex
    /// @param [in] ci2 ChunkIndex
    /// @return ci1.size_ < ci2.size_
    export bool operator< (const ChunkIndex& ci1, const ChunkIndex& ci2) {
    	return ci1.size_ < ci2.size_;
    }
    
    namespace {
    	using IT = std::vector<Chunk>::iterator;
    }
    
    /// @brief The class Node manages the allocations for a simulated NUMA node.
    ///
    ///        Each Node allocates a big block of memory, and manages simulated
    ///        NUMA allocations by itself. Each NUMA allocation is stored in
    ///        a [chunk](@ref Chunk), which contains the address and the size of
    ///        the NUMA allocation. Each used [chunk](@ref Chunk) is stored in the
    ///        vector used_, and each freed [chunk](@ref Chunk) ist stored in the
    ///        vector free_.
    ///
    export class Node : private boost::noncopyable {
    	/// The size of the simulated NUMA node memory.
    	size_t size_;
    
    	std::vector<Chunk> free_;
    	std::vector<Chunk> used_;
    
    	/// The pointer to the allocated simulated NUMA memory.
    	std::unique_ptr<char, decltype (&free)> p_;
    public:
    	/// The MemoryPool class allocates a array of Nodes, so we cannot initialize
    	/// the instances of class Node directly via the constructor.
    	/// @post Both vectors are default constructed, size_ is set to zero and p_ is a nullptr.
    	/// 
    	Node () : size_(0), free_({}), used_({}), p_(nullptr, free) {}
    
    	/// @brief Allocates the memory for the simulated NUMA node.
    	/// @param [in] size Size of the simulated NUMA memory space.
    	/// @pre size_ is zero and p_ is a nullptr.
    	/// @post The memory is allocated and setup of Node is completed.
    	/// @exception If size is zero, or [node](@ref Node) is allready
    	///            initialized, or if the memory block is to large, then
    	///            std::bad_alloc is thrown.
    	/// If the init function is called a second time, std::runtime_error is thrown.
    	void init (const size_t size) {
    		void* temp_pointer = nullptr;
    
    		// check the pre conditions and if the malloc is successfull
    		if (0 != size_ || 0 == size) throw std::bad_alloc();
    		if (nullptr == (temp_pointer = malloc(size))) throw std::bad_alloc();
    
    		// store pointer and other stuff to meet post conditions
    		this->p_ = std::unique_ptr<char, decltype (&std::free)> (static_cast<char*>(temp_pointer), free);
    		size_ = size;
    
    		Chunk ck (size, temp_pointer);
    		this->free_.push_back(ck);
    	}
    
    	/// \brief Helper function which calculates a new pointer.
    	/// \param [in] pointer The pointer to the allocated memory block.
    	/// \param [in] size Size if allocated memory block.
    	/// \return New pointer, which points to p+s.
    	[[nodiscard]] void* generate_pointer_with_offset (void* const pointer, const size_t size) noexcept {
    		// We have to cast to char*, because void* does not allow to do pointer arithmetics.
    		char* new_pointer = static_cast<char*> (pointer);
    		new_pointer += size;
    		return static_cast<void*> (new_pointer);
    	}
    
    	[[nodiscard]] void* calculate_base_pointer (void* const pointer, const size_t size) noexcept {
    		char* base_pointer = static_cast<char*> (pointer);
    		base_pointer -= size;
    
    		return static_cast<void*> (base_pointer);
    	}
    
    	/// @brief
    	/// @param [in] size of the allocation
    	/// @pre   free_ is sorted by size of chunk.size_ 
    	/// @post 
    	[[nodiscard]] void* allocate (const size_t size) {
    		if (0 == size) return nullptr;
    		if (size_ < size) throw std::bad_alloc();
    
    		void* new_pointer = nullptr;
    		bool found = false;
    		const size_t end = free_.size();
    		std::vector<Chunk>::size_type pos = 0;
    
    		std::vector<ChunkIndex> freeIndex(free_.size());
    		for (size_t i = 0; i < end; ++i) {
    			freeIndex[i] = {free_[i].size_, i};
    		}
    		sort (freeIndex.begin(), freeIndex.end());
    
    		for (size_t i = 0; i < end; ++i) {
    			if (freeIndex[i].size_ >= size) {
    				pos = i;
    				found = true;
    				break;
    			} 
    		}
    
    		if (!found) throw std::bad_alloc();
    
    		IT it = free_.begin() + pos;
    		new_pointer = it->p_;
    		Chunk ck {size, new_pointer};
    		it->p_     = generate_pointer_with_offset (it->p_, size);
    		it->size_ -= size;
    
    		IT position = upper_bound (used_.begin(), used_.end(), ck);
    		used_.insert (position, ck);
    		if (0 == it->size_) free_.erase(it);
    
    		std::cout << " p=" << new_pointer << "\n";
    
    		return new_pointer;
    	}
    
    	/// @brief 
    	/// @param [in,out] pointer
    	/// @param [in] size
    	/// @pre The pointer p must be poiting to memory allocated for node \p node. I.e. p must be in the nodes_.used_ vector.
    	/// @post Test
    	void deallocate (void* pointer, const size_t size) {
    		if (nullptr == pointer) return;
    
    		Chunk ck = {size, pointer};
    		std::pair<IT,IT> pair = std::equal_range (used_.begin(), used_.end(), ck);
    
    		if (pair.first == used_.end()) {
    			throw std::runtime_error ("Node::deallocation failed");
    		}
    
    		used_.erase(pair.first);
    
    		IT position = std::upper_bound (free_.begin(), free_.end(), ck);
    
    		free_.insert (position, ck);
    
    		// searching for Chunks which can be merged with this chunk
    		recycle();
    	}
    
    	/// \brief This function 
    	void recycle() {
    		if (free_.size() <= 1) return;
    
    		std::vector<Chunk> fs (free_);
    		std::sort(fs.begin(), fs.end());
    
    		const size_t e = fs.size();
    
    		for (ssize_t i = (e - 2); i >= 0; --i) {
    			char* p = static_cast<char*> (fs[i].p_);
    			char* q = p + fs[i].size_;
    
    			if (q == fs[i+1].p_) {
    				fs[i].size_ += fs[i+1].size_;
    				IT it = next(fs.begin(), (i+1));
    				fs.erase(it);
    			}
    		}
    
    		swap(fs, free_);
    		sort(free_.begin(), free_.end());
    	}
    };
    
    export class MemoryPool : private boost::noncopyable {
    private:
    	int			n_nodes_;
    	std::unique_ptr<Node[]>	nodes_;
    public:
    	~MemoryPool () = default;
    
    	/// \brief This constructor 
    	/// \param [in] number_nodes The number of NUMA nodes to simulate.
    	/// \param [in] size The memory size of each simulated NUMA node.
    	MemoryPool (const int number_nodes, const size_t size) : n_nodes_(number_nodes), nodes_(new Node[number_nodes]) {
    		for (int i = 0; i < n_nodes_; ++i) {
    			this->nodes_[i].init(size);
    		}
    	}
    
    	/// \brief This function allocates @p n memory on simulated NUMA node @p node.
    	/// \param [in] size
    	/// \param [in] node
    	/// \return Pointer to newly allocated memory.
    	/// @exception If node does not reference an simulated NUMA node or
    	[[nodiscard]] void* allocate (const size_t size, const int node) {
    		std::cout << "MemoryPool::allocate   node=" << node << " size=" << size;
    		if ((0 > node) || (n_nodes_ <= node)) throw std::runtime_error ("allocate failed; node number is incorrect");
    
    		return this->nodes_[node].allocate (size);
    	}
    
    	/// \brief
    	/// @param [in] pointer
    	/// @param [in] size
    	/// @param [in] node
    	void deallocate (void* pointer, const size_t size, const int node) {
    		std::cout << "MemoryPool::deallocate node=" << node << " size=" << size << " p=" << pointer << std::endl;
    		this->nodes_[node].deallocate(pointer, size);
    	}
    };
    
    /// \brief This allocator class simulates
    /// \tparam T any possible type
    export template <typename T>
    class NUMA_alloc_sim {
    		int node_;
    		std::shared_ptr<MemoryPool> smp_;
    	public:
    		using value_type 		= T;
    		using size_type			= size_t;
    
    		// The new approach for allocators is they do not define these values,
    		// if they are false. So for testing purposes, these lines are present
    		// but not used.
    		//using propagate_on_container_copy_assignment	= std::false_type;
    		//using propagate_on_container_move_assignment	= std::false_type;
    		//using propagate_on_container_swap				= std::false_type;
    		//using is_always_equal							= std::false_type;
    
    		// 
    		~NUMA_alloc_sim() = default;
    		/// @param [in] node ssss
    		/// @param [in] smp
    		NUMA_alloc_sim (const int node, std::shared_ptr<MemoryPool> smp) noexcept : node_(node), smp_(smp) {}
    		NUMA_alloc_sim () noexcept : node_(0), smp_(nullptr) {}
    		NUMA_alloc_sim (const NUMA_alloc_sim& rhs) noexcept = default;
    		template <class U> NUMA_alloc_sim (const NUMA_alloc_sim<U>& rhs) noexcept;
    		NUMA_alloc_sim (NUMA_alloc_sim&&) = default;
    		NUMA_alloc_sim& operator= (const NUMA_alloc_sim&) = default;
    		NUMA_alloc_sim& operator= (NUMA_alloc_sim&&) = default;
    
    		[[nodiscard]] inline T* allocate (const size_type n) {
    			return static_cast<T*> (this->smp_->allocate(sizeof(T)*n, this->node_));
    		}
    		inline void deallocate (T* p, const size_type n) {
    			this->smp_->deallocate(p, n*sizeof(T), this->node_);
    		}
    		template <typename U> friend bool operator== (const NUMA_alloc_sim<U>&, const NUMA_alloc_sim<U>&);
    		template <typename U> friend bool operator!= (const NUMA_alloc_sim<U>&, const NUMA_alloc_sim<U>&);
    		template <typename U, typename V> friend constexpr bool operator== (const NUMA_alloc_sim<U>& lhs, const NUMA_alloc_sim<V>& rhs);
    		template <typename U, typename V> friend constexpr bool operator!= (const NUMA_alloc_sim<U>& lhs, const NUMA_alloc_sim<V>& rhs);
    };
    
    export template <typename U>
    bool operator== (const NUMA_alloc_sim<U>& lhs, const NUMA_alloc_sim<U>& rhs) {
    	return (lhs.node_ == rhs.node_);
    }
    
    export template <typename U>
    bool operator!= (const NUMA_alloc_sim<U>& lhs, const NUMA_alloc_sim<U>& rhs) {
    	return (lhs.node_ != rhs.node_);
    }
    
    export template <class U, class V>
    constexpr bool operator== (const NUMA_alloc_sim<U>& lhs, const NUMA_alloc_sim<V>& rhs) {
    	return false;
    }
    
    export template <class U, class V>
    constexpr bool operator!= (const NUMA_alloc_sim<U>& lhs, const NUMA_alloc_sim<V>& rhs) {
    	return true;
    }
    

    und die Fehlermeldung

    g++-11 -std=c++20 -O2 -fmodules-ts -c NUMA_Allocator_Simulator.cc
    In Modul, importiert bei NUMA_Allocator_Simulator.cc:12:1:
    /opt/gcc-11.1.0/include/c++/11.1.0/memory: Anmerkung: unable to represent further imported source locations
    NUMA_Allocator_Simulator.cc: In Elementfunktion »void Node::init(size_t)«:
    NUMA_Allocator_Simulator.cc:96:70: Fehler: »decltype« kann die Adresse der überladenen Funktion nicht auflösen
       96 |                 this->p_ = std::unique_ptr<char, decltype (&std::free)> (static_cast<char*>(temp_pointer), free);
          |                                                                      ^
    NUMA_Allocator_Simulator.cc:96:71: Fehler: Templateargument 2 ist ungültig
       96 |                 this->p_ = std::unique_ptr<char, decltype (&std::free)> (static_cast<char*>(temp_pointer), free);
          |                                                                       ^
    NUMA_Allocator_Simulator.cc: Im globalen Gültigkeitsbereich:
    NUMA_Allocator_Simulator.cc:8:8: Warnung: not writing module »NUMA_Allocator_Simulator« due to errors
        8 | export module NUMA_Allocator_Simulator;
          |        ^~~~~~
    make: *** [makefile:140: gcm.cache/NUMA_Allocator_Simulator.gcm] Fehler 1
    

    Frage:
    Offentsichtlich findet er die passende Funktion nicht, was aber beim Übersetzen des Modules ja eigentlich kein Problem sein sollte so mein Verständnis, da das Module keinen fertigen Programmcode enthält sondern nur präcompilierten Code des Compilers, der erst später in vollständigen Code gewandelt wird. Ist das nun ein Problem, weil man generell so in Modulen nicht mehr so machen kann, oder ist das nur ein Problem von g++ 11.1.0 … oder?



  • Warum benutzt du dort std::free und sonst nur free?



  • @john-0 sagte in GCC 11.1.0 Module und ein Fehler:

    das Module keinen fertigen Programmcode enthält sondern nur präcompilierten Code des Compilers, der erst später in vollständigen Code gewandelt wird

    Mit

    g++-11 -std=c++20 -O2 -fmodules-ts -c NUMA_Allocator_Simulator.cc

    wird ganz normal übersetzt. Du musst vorher mit den passenden Flags den „präcompilierten“ Code erstellen. Ohne Buildsystem ist das eher mühselig. build2 soll mit gcc11 + Modulen grundsätzlich zusammenarbeiten können, für make hat mal jemand etwas ausprobiert, aber das ist wohl nur in einem Branch.

    Keine Ahnung, ob das mit dem Fehler zusammenhängt.



  • @Th69 sagte in GCC 11.1.0 Module und ein Fehler:

    Warum benutzt du dort std::free und sonst nur free?

    Autsch, manchmal sieht man den Wald vor lauter Bäumen nicht. Jetzt geht es etwas weiter und es gibt dann die nächste Fehlermeldung, und die sieht nicht gut aus.

    g++-11 -std=c++20 -O2 -fmodules-ts -c NUMA_Allocator_Simulator.cc
    In Modul, importiert bei NUMA_Allocator_Simulator.cc:12:1:
    /opt/gcc-11.1.0/include/c++/11.1.0/memory: Anmerkung: unable to represent further imported source locations
    NUMA_Allocator_Simulator.cc:8:8: interner Compiler-Fehler: in write_location, bei cp/module.cc:15605
        8 | export module NUMA_Allocator_Simulator;
          |        ^~~~~~
    0x6b58be module_state::write_location(bytes_out&, unsigned int)
            ../../gcc-11.1.0/gcc/cp/module.cc:15605
    0x6bbd05 trees_out::core_vals(tree_node*)
            ../../gcc-11.1.0/gcc/cp/module.cc:5798
    0x6bc7ce trees_out::tree_node_vals(tree_node*)
            ../../gcc-11.1.0/gcc/cp/module.cc:7052
    0x6bca41 trees_out::tpl_parm_value(tree_node*)
            ../../gcc-11.1.0/gcc/cp/module.cc:7325
    0x6bcd8c trees_out::tree_node(tree_node*)
            ../../gcc-11.1.0/gcc/cp/module.cc:9046
    0x6bdb9a trees_out::type_node(tree_node*)
            ../../gcc-11.1.0/gcc/cp/module.cc:8678
    0x6bccce trees_out::tree_node(tree_node*)
            ../../gcc-11.1.0/gcc/cp/module.cc:9038
    0x6bc4af trees_out::core_vals(tree_node*)
            ../../gcc-11.1.0/gcc/cp/module.cc:6143
    0x6bc7ce trees_out::tree_node_vals(tree_node*)
            ../../gcc-11.1.0/gcc/cp/module.cc:7052
    0x6bd19d trees_out::tree_value(tree_node*)
            ../../gcc-11.1.0/gcc/cp/module.cc:8888
    0x6bcdec trees_out::tree_node(tree_node*)
            ../../gcc-11.1.0/gcc/cp/module.cc:9086
    0x6bd2cc trees_out::key_mergeable(int, merge_kind, tree_node*, tree_node*, tree_node*, depset*)
            ../../gcc-11.1.0/gcc/cp/module.cc:10308
    0x6bea43 trees_out::decl_value(tree_node*, depset*)
            ../../gcc-11.1.0/gcc/cp/module.cc:7620
    0x6befb5 trees_out::decl_node(tree_node*, walk_kind)
            ../../gcc-11.1.0/gcc/cp/module.cc:8243
    0x6bcddd trees_out::tree_node(tree_node*)
            ../../gcc-11.1.0/gcc/cp/module.cc:9081
    0x6bfe6f module_state::write_cluster(elf_out*, depset**, unsigned int, depset::hash&, unsigned int*, unsigned int*)
            ../../gcc-11.1.0/gcc/cp/module.cc:14629
    0x6c89cc module_state::write(elf_out*, cpp_reader*)
            ../../gcc-11.1.0/gcc/cp/module.cc:17732
    0x6ad376 finish_module_processing(cpp_reader*)
            ../../gcc-11.1.0/gcc/cp/module.cc:19855
    0x10a6411 c_parse_final_cleanups()
            ../../gcc-11.1.0/gcc/cp/decl2.c:5175
    Bitte senden Sie einen vollständigen Fehlerbericht auf Englisch ein;
    inclusive vorverarbeitetem Quellcode, wenn es dienlich ist.
    Please include the complete backtrace with any bug report.
    Weitere Hinweise finden Sie unter »<https://gcc.gnu.org/bugs/>«.
    make: *** [makefile:140: gcm.cache/NUMA_Allocator_Simulator.gcm] Fehler 1
    


  • @manni66 sagte in GCC 11.1.0 Module und ein Fehler:

    Ohne Buildsystem ist das eher mühselig. build2 soll mit gcc11 + Modulen grundsätzlich zusammenarbeiten können, für make hat mal jemand etwas ausprobiert, aber das ist wohl nur in einem Branch.

    Es funktioniert mit GNU make ganz gut, aber war einiges an Arbeit im Makefile das umzustellen. Die Regeln muss man sich per foreach und eval generieren lassen und dann erkennt er auch die Abhängigkeiten richtig und baut alles bis zum Fehler korrekt.