Blaze 3.9
Dense.h
Go to the documentation of this file.
1//=================================================================================================
33//=================================================================================================
34
35#ifndef _BLAZE_MATH_VIEWS_SUBMATRIX_DENSE_H_
36#define _BLAZE_MATH_VIEWS_SUBMATRIX_DENSE_H_
37
38
39//*************************************************************************************************
40// Includes
41//*************************************************************************************************
42
43#include <algorithm>
44#include <iterator>
45#include <blaze/math/Aliases.h>
65#include <blaze/math/SIMD.h>
96#include <blaze/system/Inline.h>
102#include <blaze/util/Assert.h>
106#include <blaze/util/EnableIf.h>
108#include <blaze/util/mpl/If.h>
109#include <blaze/util/Types.h>
112
113
114namespace blaze {
115
116//=================================================================================================
117//
118// CLASS TEMPLATE SPECIALIZATION FOR UNALIGNED ROW-MAJOR DENSE SUBMATRICES
119//
120//=================================================================================================
121
122//*************************************************************************************************
130template< typename MT // Type of the dense matrix
131 , size_t... CSAs > // Compile time submatrix arguments
132class Submatrix<MT,unaligned,false,true,CSAs...>
133 : public View< DenseMatrix< Submatrix<MT,unaligned,false,true,CSAs...>, false > >
134 , private SubmatrixData<CSAs...>
135{
136 private:
137 //**Type definitions****************************************************************************
138 using DataType = SubmatrixData<CSAs...>;
139 using Operand = If_t< IsExpression_v<MT>, MT, MT& >;
140 //**********************************************************************************************
141
142 //**********************************************************************************************
144 template< typename MT1, typename MT2 >
145 static constexpr bool EnforceEvaluation_v =
146 ( IsRestricted_v<MT1> && RequiresEvaluation_v<MT2> );
147 //**********************************************************************************************
148
149 public:
150 //**Type definitions****************************************************************************
152 using This = Submatrix<MT,unaligned,false,true,CSAs...>;
153
155 using BaseType = View< DenseMatrix<This,false> >;
156
157 using ViewedType = MT;
158 using ResultType = SubmatrixTrait_t<MT,CSAs...>;
159 using OppositeType = OppositeType_t<ResultType>;
160 using TransposeType = TransposeType_t<ResultType>;
161 using ElementType = ElementType_t<MT>;
162 using SIMDType = SIMDTrait_t<ElementType>;
163 using ReturnType = ReturnType_t<MT>;
164 using CompositeType = const Submatrix&;
165
167 using ConstReference = ConstReference_t<MT>;
168
170 using Reference = If_t< IsConst_v<MT>, ConstReference, Reference_t<MT> >;
171
173 using ConstPointer = ConstPointer_t<MT>;
174
176 using Pointer = If_t< IsConst_v<MT> || !HasMutableDataAccess_v<MT>, ConstPointer, Pointer_t<MT> >;
177 //**********************************************************************************************
178
179 //**SubmatrixIterator class definition**********************************************************
182 template< typename IteratorType > // Type of the dense matrix iterator
183 class SubmatrixIterator
184 {
185 public:
186 //**Type definitions*************************************************************************
188 using IteratorCategory = typename std::iterator_traits<IteratorType>::iterator_category;
189
191 using ValueType = typename std::iterator_traits<IteratorType>::value_type;
192
194 using PointerType = typename std::iterator_traits<IteratorType>::pointer;
195
197 using ReferenceType = typename std::iterator_traits<IteratorType>::reference;
198
200 using DifferenceType = typename std::iterator_traits<IteratorType>::difference_type;
201
202 // STL iterator requirements
203 using iterator_category = IteratorCategory;
204 using value_type = ValueType;
205 using pointer = PointerType;
206 using reference = ReferenceType;
207 using difference_type = DifferenceType;
208 //*******************************************************************************************
209
210 //**Constructor******************************************************************************
213 inline SubmatrixIterator()
214 : iterator_ ( ) // Iterator to the current submatrix element
215 , isAligned_( false ) // Memory alignment flag
216 {}
217 //*******************************************************************************************
218
219 //**Constructor******************************************************************************
225 inline SubmatrixIterator( IteratorType iterator, bool isMemoryAligned )
226 : iterator_ ( iterator ) // Iterator to the current submatrix element
227 , isAligned_( isMemoryAligned ) // Memory alignment flag
228 {}
229 //*******************************************************************************************
230
231 //**Constructor******************************************************************************
236 template< typename IteratorType2 >
237 inline SubmatrixIterator( const SubmatrixIterator<IteratorType2>& it )
238 : iterator_ ( it.base() ) // Iterator to the current submatrix element
239 , isAligned_( it.isAligned() ) // Memory alignment flag
240 {}
241 //*******************************************************************************************
242
243 //**Addition assignment operator*************************************************************
249 inline SubmatrixIterator& operator+=( size_t inc ) {
250 iterator_ += inc;
251 return *this;
252 }
253 //*******************************************************************************************
254
255 //**Subtraction assignment operator**********************************************************
261 inline SubmatrixIterator& operator-=( size_t dec ) {
262 iterator_ -= dec;
263 return *this;
264 }
265 //*******************************************************************************************
266
267 //**Prefix increment operator****************************************************************
272 inline SubmatrixIterator& operator++() {
273 ++iterator_;
274 return *this;
275 }
276 //*******************************************************************************************
277
278 //**Postfix increment operator***************************************************************
283 inline const SubmatrixIterator operator++( int ) {
284 return SubmatrixIterator( iterator_++, isAligned_ );
285 }
286 //*******************************************************************************************
287
288 //**Prefix decrement operator****************************************************************
293 inline SubmatrixIterator& operator--() {
294 --iterator_;
295 return *this;
296 }
297 //*******************************************************************************************
298
299 //**Postfix decrement operator***************************************************************
304 inline const SubmatrixIterator operator--( int ) {
305 return SubmatrixIterator( iterator_--, isAligned_ );
306 }
307 //*******************************************************************************************
308
309 //**Element access operator******************************************************************
314 inline ReferenceType operator*() const {
315 return *iterator_;
316 }
317 //*******************************************************************************************
318
319 //**Element access operator******************************************************************
324 inline IteratorType operator->() const {
325 return iterator_;
326 }
327 //*******************************************************************************************
328
329 //**Load function****************************************************************************
339 inline SIMDType load() const noexcept {
340 if( isAligned_ )
341 return loada();
342 else
343 return loadu();
344 }
345 //*******************************************************************************************
346
347 //**Loada function***************************************************************************
357 inline SIMDType loada() const noexcept {
358 return iterator_.loada();
359 }
360 //*******************************************************************************************
361
362 //**Loadu function***************************************************************************
372 inline SIMDType loadu() const noexcept {
373 return iterator_.loadu();
374 }
375 //*******************************************************************************************
376
377 //**Store function***************************************************************************
388 inline void store( const SIMDType& value ) const {
389 if( isAligned_ ) {
390 storea( value );
391 }
392 else {
393 storeu( value );
394 }
395 }
396 //*******************************************************************************************
397
398 //**Storea function**************************************************************************
409 inline void storea( const SIMDType& value ) const {
410 iterator_.storea( value );
411 }
412 //*******************************************************************************************
413
414 //**Storeu function**************************************************************************
425 inline void storeu( const SIMDType& value ) const {
426 iterator_.storeu( value );
427 }
428 //*******************************************************************************************
429
430 //**Stream function**************************************************************************
441 inline void stream( const SIMDType& value ) const {
442 iterator_.stream( value );
443 }
444 //*******************************************************************************************
445
446 //**Equality operator************************************************************************
452 inline bool operator==( const SubmatrixIterator& rhs ) const {
453 return iterator_ == rhs.iterator_;
454 }
455 //*******************************************************************************************
456
457 //**Inequality operator**********************************************************************
463 inline bool operator!=( const SubmatrixIterator& rhs ) const {
464 return iterator_ != rhs.iterator_;
465 }
466 //*******************************************************************************************
467
468 //**Less-than operator***********************************************************************
474 inline bool operator<( const SubmatrixIterator& rhs ) const {
475 return iterator_ < rhs.iterator_;
476 }
477 //*******************************************************************************************
478
479 //**Greater-than operator********************************************************************
485 inline bool operator>( const SubmatrixIterator& rhs ) const {
486 return iterator_ > rhs.iterator_;
487 }
488 //*******************************************************************************************
489
490 //**Less-or-equal-than operator**************************************************************
496 inline bool operator<=( const SubmatrixIterator& rhs ) const {
497 return iterator_ <= rhs.iterator_;
498 }
499 //*******************************************************************************************
500
501 //**Greater-or-equal-than operator***********************************************************
507 inline bool operator>=( const SubmatrixIterator& rhs ) const {
508 return iterator_ >= rhs.iterator_;
509 }
510 //*******************************************************************************************
511
512 //**Subtraction operator*********************************************************************
518 inline DifferenceType operator-( const SubmatrixIterator& rhs ) const {
519 return iterator_ - rhs.iterator_;
520 }
521 //*******************************************************************************************
522
523 //**Addition operator************************************************************************
530 friend inline const SubmatrixIterator operator+( const SubmatrixIterator& it, size_t inc ) {
531 return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
532 }
533 //*******************************************************************************************
534
535 //**Addition operator************************************************************************
542 friend inline const SubmatrixIterator operator+( size_t inc, const SubmatrixIterator& it ) {
543 return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
544 }
545 //*******************************************************************************************
546
547 //**Subtraction operator*********************************************************************
554 friend inline const SubmatrixIterator operator-( const SubmatrixIterator& it, size_t dec ) {
555 return SubmatrixIterator( it.iterator_ - dec, it.isAligned_ );
556 }
557 //*******************************************************************************************
558
559 //**Base function****************************************************************************
564 inline IteratorType base() const {
565 return iterator_;
566 }
567 //*******************************************************************************************
568
569 //**IsAligned function***********************************************************************
574 inline bool isAligned() const noexcept {
575 return isAligned_;
576 }
577 //*******************************************************************************************
578
579 private:
580 //**Member variables*************************************************************************
581 IteratorType iterator_;
582 bool isAligned_;
583 //*******************************************************************************************
584 };
585 //**********************************************************************************************
586
587 //**Type definitions****************************************************************************
589 using ConstIterator = SubmatrixIterator< ConstIterator_t<MT> >;
590
592 using Iterator = If_t< IsConst_v<MT>, ConstIterator, SubmatrixIterator< Iterator_t<MT> > >;
593 //**********************************************************************************************
594
595 //**Compilation flags***************************************************************************
597 static constexpr bool simdEnabled = MT::simdEnabled;
598
600 static constexpr bool smpAssignable = MT::smpAssignable;
601
603 static constexpr bool compileTimeArgs = DataType::compileTimeArgs;
604 //**********************************************************************************************
605
606 //**Constructors********************************************************************************
609 template< typename... RSAs >
610 explicit inline Submatrix( MT& matrix, RSAs... args );
611
612 Submatrix( const Submatrix& ) = default;
614 //**********************************************************************************************
615
616 //**Destructor**********************************************************************************
619 ~Submatrix() = default;
621 //**********************************************************************************************
622
623 //**Data access functions***********************************************************************
626 inline Reference operator()( size_t i, size_t j );
627 inline ConstReference operator()( size_t i, size_t j ) const;
628 inline Reference at( size_t i, size_t j );
629 inline ConstReference at( size_t i, size_t j ) const;
630 inline Pointer data () noexcept;
631 inline ConstPointer data () const noexcept;
632 inline Pointer data ( size_t i ) noexcept;
633 inline ConstPointer data ( size_t i ) const noexcept;
634 inline Iterator begin ( size_t i );
635 inline ConstIterator begin ( size_t i ) const;
636 inline ConstIterator cbegin( size_t i ) const;
637 inline Iterator end ( size_t i );
638 inline ConstIterator end ( size_t i ) const;
639 inline ConstIterator cend ( size_t i ) const;
641 //**********************************************************************************************
642
643 //**Assignment operators************************************************************************
646 inline Submatrix& operator=( const ElementType& rhs );
647 inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
648 inline Submatrix& operator=( const Submatrix& rhs );
649
650 template< typename MT2, bool SO2 >
651 inline Submatrix& operator=( const Matrix<MT2,SO2>& rhs );
652
653 template< typename MT2, bool SO2 >
654 inline auto operator+=( const Matrix<MT2,SO2>& rhs )
655 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
656
657 template< typename MT2, bool SO2 >
658 inline auto operator+=( const Matrix<MT2,SO2>& rhs )
659 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
660
661 template< typename MT2, bool SO2 >
662 inline auto operator-=( const Matrix<MT2,SO2>& rhs )
663 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
664
665 template< typename MT2, bool SO2 >
666 inline auto operator-=( const Matrix<MT2,SO2>& rhs )
667 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
668
669 template< typename MT2, bool SO2 >
670 inline auto operator%=( const Matrix<MT2,SO2>& rhs )
671 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
672
673 template< typename MT2, bool SO2 >
674 inline auto operator%=( const Matrix<MT2,SO2>& rhs )
675 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
677 //**********************************************************************************************
678
679 //**Utility functions***************************************************************************
682 using DataType::row;
683 using DataType::column;
684 using DataType::rows;
685 using DataType::columns;
686
687 inline MT& operand() noexcept;
688 inline const MT& operand() const noexcept;
689
690 inline size_t spacing() const noexcept;
691 inline size_t capacity() const noexcept;
692 inline size_t capacity( size_t i ) const noexcept;
693 inline size_t nonZeros() const;
694 inline size_t nonZeros( size_t i ) const;
695 inline void reset();
696 inline void reset( size_t i );
698 //**********************************************************************************************
699
700 //**Numeric functions***************************************************************************
703 inline Submatrix& transpose();
704 inline Submatrix& ctranspose();
705
706 template< typename Other > inline Submatrix& scale( const Other& scalar );
708 //**********************************************************************************************
709
710 private:
711 //**********************************************************************************************
713 template< typename MT2 >
714 static constexpr bool VectorizedAssign_v =
715 ( useOptimizedKernels &&
716 simdEnabled && MT2::simdEnabled &&
717 IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > );
718 //**********************************************************************************************
719
720 //**********************************************************************************************
722 template< typename MT2 >
723 static constexpr bool VectorizedAddAssign_v =
724 ( VectorizedAssign_v<MT2> &&
725 HasSIMDAdd_v< ElementType, ElementType_t<MT2> > &&
726 !IsDiagonal_v<MT2> );
727 //**********************************************************************************************
728
729 //**********************************************************************************************
731 template< typename MT2 >
732 static constexpr bool VectorizedSubAssign_v =
733 ( VectorizedAssign_v<MT2> &&
734 HasSIMDSub_v< ElementType, ElementType_t<MT2> > &&
735 !IsDiagonal_v<MT2> );
736 //**********************************************************************************************
737
738 //**********************************************************************************************
740 template< typename MT2 >
741 static constexpr bool VectorizedSchurAssign_v =
742 ( VectorizedAssign_v<MT2> &&
743 HasSIMDMult_v< ElementType, ElementType_t<MT2> > );
744 //**********************************************************************************************
745
746 //**SIMD properties*****************************************************************************
748 static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
749 //**********************************************************************************************
750
751 public:
752 //**Expression template evaluation functions****************************************************
755 template< typename Other >
756 inline bool canAlias( const Other* alias ) const noexcept;
757
758 template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
759 inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
760
761 template< typename Other >
762 inline bool isAliased( const Other* alias ) const noexcept;
763
764 template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
765 inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
766
767 inline bool isAligned () const noexcept;
768 inline bool canSMPAssign() const noexcept;
769
770 BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
771 BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
772 BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
773
774 BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
775 BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
776 BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
777 BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
778
779 template< typename MT2 >
780 inline auto assign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT2> >;
781
782 template< typename MT2 >
783 inline auto assign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT2> >;
784
785 template< typename MT2 > inline void assign( const DenseMatrix<MT2,true>& rhs );
786 template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
787 template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
788
789 template< typename MT2 >
790 inline auto addAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT2> >;
791
792 template< typename MT2 >
793 inline auto addAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT2> >;
794
795 template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,true>& rhs );
796 template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
797 template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
798
799 template< typename MT2 >
800 inline auto subAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT2> >;
801
802 template< typename MT2 >
803 inline auto subAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT2> >;
804
805 template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,true>& rhs );
806 template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
807 template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
808
809 template< typename MT2 >
810 inline auto schurAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT2> >;
811
812 template< typename MT2 >
813 inline auto schurAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT2> >;
814
815 template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,true>& rhs );
816 template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
817 template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
819 //**********************************************************************************************
820
821 private:
822 //**Utility functions***************************************************************************
825 inline bool hasOverlap() const noexcept;
827 //**********************************************************************************************
828
829 //**Member variables****************************************************************************
832 Operand matrix_;
833 const bool isAligned_;
841 //**********************************************************************************************
842
843 //**Friend declarations*************************************************************************
844 template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
845 //**********************************************************************************************
846
847 //**Compile time checks*************************************************************************
855 //**********************************************************************************************
856};
858//*************************************************************************************************
859
860
861
862
863//=================================================================================================
864//
865// CONSTRUCTORS
866//
867//=================================================================================================
868
869//*************************************************************************************************
882template< typename MT // Type of the dense matrix
883 , size_t... CSAs > // Compile time submatrix arguments
884template< typename... RSAs > // Runtime submatrix arguments
885inline Submatrix<MT,unaligned,false,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
886 : DataType ( args... ) // Base class initialization
887 , matrix_ ( matrix ) // The matrix containing the submatrix
888 , isAligned_( simdEnabled && IsContiguous_v<MT> &&
889 matrix.data() != nullptr && checkAlignment( data() ) &&
890 ( rows() < 2UL || ( matrix.spacing() % SIMDSIZE ) == 0UL ) )
891{
892 if( isChecked( args... ) ) {
893 if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
894 BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
895 }
896 }
897 else {
898 BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
899 BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
900 }
901}
903//*************************************************************************************************
904
905
906
907
908//=================================================================================================
909//
910// DATA ACCESS FUNCTIONS
911//
912//=================================================================================================
913
914//*************************************************************************************************
925template< typename MT // Type of the dense matrix
926 , size_t... CSAs > // Compile time submatrix arguments
927inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Reference
928 Submatrix<MT,unaligned,false,true,CSAs...>::operator()( size_t i, size_t j )
929{
930 BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
931 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
932
933 return matrix_(row()+i,column()+j);
934}
936//*************************************************************************************************
937
938
939//*************************************************************************************************
950template< typename MT // Type of the dense matrix
951 , size_t... CSAs > // Compile time submatrix arguments
952inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstReference
953 Submatrix<MT,unaligned,false,true,CSAs...>::operator()( size_t i, size_t j ) const
954{
955 BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
956 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
957
958 return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
959}
961//*************************************************************************************************
962
963
964//*************************************************************************************************
976template< typename MT // Type of the dense matrix
977 , size_t... CSAs > // Compile time submatrix arguments
978inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Reference
979 Submatrix<MT,unaligned,false,true,CSAs...>::at( size_t i, size_t j )
980{
981 if( i >= rows() ) {
982 BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
983 }
984 if( j >= columns() ) {
985 BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
986 }
987 return (*this)(i,j);
988}
990//*************************************************************************************************
991
992
993//*************************************************************************************************
1005template< typename MT // Type of the dense matrix
1006 , size_t... CSAs > // Compile time submatrix arguments
1007inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstReference
1008 Submatrix<MT,unaligned,false,true,CSAs...>::at( size_t i, size_t j ) const
1009{
1010 if( i >= rows() ) {
1011 BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
1012 }
1013 if( j >= columns() ) {
1014 BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
1015 }
1016 return (*this)(i,j);
1017}
1019//*************************************************************************************************
1020
1021
1022//*************************************************************************************************
1032template< typename MT // Type of the dense matrix
1033 , size_t... CSAs > // Compile time submatrix arguments
1034inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Pointer
1036{
1037 return matrix_.data() + row()*spacing() + column();
1038}
1040//*************************************************************************************************
1041
1042
1043//*************************************************************************************************
1053template< typename MT // Type of the dense matrix
1054 , size_t... CSAs > // Compile time submatrix arguments
1055inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstPointer
1057{
1058 return matrix_.data() + row()*spacing() + column();
1059}
1061//*************************************************************************************************
1062
1063
1064//*************************************************************************************************
1073template< typename MT // Type of the dense matrix
1074 , size_t... CSAs > // Compile time submatrix arguments
1075inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Pointer
1077{
1078 return matrix_.data() + (row()+i)*spacing() + column();
1079}
1081//*************************************************************************************************
1082
1083
1084//*************************************************************************************************
1093template< typename MT // Type of the dense matrix
1094 , size_t... CSAs > // Compile time submatrix arguments
1095inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstPointer
1096 Submatrix<MT,unaligned,false,true,CSAs...>::data( size_t i ) const noexcept
1097{
1098 return matrix_.data() + (row()+i)*spacing() + column();
1099}
1101//*************************************************************************************************
1102
1103
1104//*************************************************************************************************
1116template< typename MT // Type of the dense matrix
1117 , size_t... CSAs > // Compile time submatrix arguments
1118inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Iterator
1120{
1121 BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1122 return Iterator( matrix_.begin( row() + i ) + column(), isAligned_ );
1123}
1125//*************************************************************************************************
1126
1127
1128//*************************************************************************************************
1140template< typename MT // Type of the dense matrix
1141 , size_t... CSAs > // Compile time submatrix arguments
1142inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1144{
1145 BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1146 return ConstIterator( matrix_.cbegin( row() + i ) + column(), isAligned_ );
1147}
1149//*************************************************************************************************
1150
1151
1152//*************************************************************************************************
1164template< typename MT // Type of the dense matrix
1165 , size_t... CSAs > // Compile time submatrix arguments
1166inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1168{
1169 BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1170 return ConstIterator( matrix_.cbegin( row() + i ) + column(), isAligned_ );
1171}
1173//*************************************************************************************************
1174
1175
1176//*************************************************************************************************
1188template< typename MT // Type of the dense matrix
1189 , size_t... CSAs > // Compile time submatrix arguments
1190inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Iterator
1192{
1193 BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1194 return Iterator( matrix_.begin( row() + i ) + column() + columns(), isAligned_ );
1195}
1197//*************************************************************************************************
1198
1199
1200//*************************************************************************************************
1212template< typename MT // Type of the dense matrix
1213 , size_t... CSAs > // Compile time submatrix arguments
1214inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1216{
1217 BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1218 return ConstIterator( matrix_.cbegin( row() + i ) + column() + columns(), isAligned_ );
1219}
1221//*************************************************************************************************
1222
1223
1224//*************************************************************************************************
1236template< typename MT // Type of the dense matrix
1237 , size_t... CSAs > // Compile time submatrix arguments
1238inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1240{
1241 BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1242 return ConstIterator( matrix_.cbegin( row() + i ) + column() + columns(), isAligned_ );
1243}
1245//*************************************************************************************************
1246
1247
1248
1249
1250//=================================================================================================
1251//
1252// ASSIGNMENT OPERATORS
1253//
1254//=================================================================================================
1255
1256//*************************************************************************************************
1267template< typename MT // Type of the dense matrix
1268 , size_t... CSAs > // Compile time submatrix arguments
1269inline Submatrix<MT,unaligned,false,true,CSAs...>&
1270 Submatrix<MT,unaligned,false,true,CSAs...>::operator=( const ElementType& rhs )
1271{
1272 const size_t iend( row() + rows() );
1273 decltype(auto) left( derestrict( matrix_ ) );
1274
1275 for( size_t i=row(); i<iend; ++i )
1276 {
1277 const size_t jbegin( ( IsUpper_v<MT> )
1278 ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
1279 ?( max( i+1UL, column() ) )
1280 :( max( i, column() ) ) )
1281 :( column() ) );
1282 const size_t jend ( ( IsLower_v<MT> )
1283 ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
1284 ?( min( i, column()+columns() ) )
1285 :( min( i+1UL, column()+columns() ) ) )
1286 :( column()+columns() ) );
1287
1288 for( size_t j=jbegin; j<jend; ++j ) {
1289 if( !IsRestricted_v<MT> || IsTriangular_v<MT> || trySet( matrix_, i, j, rhs ) )
1290 left(i,j) = rhs;
1291 }
1292 }
1293
1294 return *this;
1295}
1297//*************************************************************************************************
1298
1299
1300//*************************************************************************************************
1316template< typename MT // Type of the dense matrix
1317 , size_t... CSAs > // Compile time submatrix arguments
1318inline Submatrix<MT,unaligned,false,true,CSAs...>&
1319 Submatrix<MT,unaligned,false,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
1320{
1321 if( list.size() != rows() ) {
1322 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
1323 }
1324
1325 if( IsRestricted_v<MT> ) {
1326 const InitializerMatrix<ElementType> tmp( list, columns() );
1327 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1328 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1329 }
1330 }
1331
1332 decltype(auto) left( derestrict( *this ) );
1333 size_t i( 0UL );
1334
1335 for( const auto& rowList : list ) {
1336 std::fill( std::copy( rowList.begin(), rowList.end(), left.begin(i) ), left.end(i), ElementType() );
1337 ++i;
1338 }
1339
1340 return *this;
1341}
1343//*************************************************************************************************
1344
1345
1346//*************************************************************************************************
1361template< typename MT // Type of the dense matrix
1362 , size_t... CSAs > // Compile time submatrix arguments
1363inline Submatrix<MT,unaligned,false,true,CSAs...>&
1364 Submatrix<MT,unaligned,false,true,CSAs...>::operator=( const Submatrix& rhs )
1365{
1368
1369 if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
1370 return *this;
1371
1372 if( rows() != rhs.rows() || columns() != rhs.columns() ) {
1373 BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
1374 }
1375
1376 if( !tryAssign( matrix_, rhs, row(), column() ) ) {
1377 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1378 }
1379
1380 decltype(auto) left( derestrict( *this ) );
1381
1382 if( rhs.canAlias( this ) ) {
1383 const ResultType tmp( rhs );
1384 smpAssign( left, tmp );
1385 }
1386 else {
1387 smpAssign( left, rhs );
1388 }
1389
1390 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1391
1392 return *this;
1393}
1395//*************************************************************************************************
1396
1397
1398//*************************************************************************************************
1413template< typename MT // Type of the dense matrix
1414 , size_t... CSAs > // Compile time submatrix arguments
1415template< typename MT2 // Type of the right-hand side matrix
1416 , bool SO2 > // Storage order of the right-hand side matrix
1417inline Submatrix<MT,unaligned,false,true,CSAs...>&
1418 Submatrix<MT,unaligned,false,true,CSAs...>::operator=( const Matrix<MT2,SO2>& rhs )
1419{
1421
1422 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
1423 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1424 }
1425
1426 using Right = If_t< IsRestricted_v<MT>, CompositeType_t<MT2>, const MT2& >;
1427 Right right( *rhs );
1428
1429 if( !tryAssign( matrix_, right, row(), column() ) ) {
1430 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1431 }
1432
1433 decltype(auto) left( derestrict( *this ) );
1434
1435 if( IsReference_v<Right> && right.canAlias( this ) ) {
1436 const ResultType_t<MT2> tmp( right );
1437 if( IsSparseMatrix_v<MT2> )
1438 reset();
1439 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
1440 }
1441 else {
1442 if( IsSparseMatrix_v<MT2> )
1443 reset();
1444 smpAssign( left, transIf< IsSymmetric_v<This> >( right ) );
1445 }
1446
1447 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1448
1449 return *this;
1450}
1452//*************************************************************************************************
1453
1454
1455//*************************************************************************************************
1469template< typename MT // Type of the dense matrix
1470 , size_t... CSAs > // Compile time submatrix arguments
1471template< typename MT2 // Type of the right-hand side matrix
1472 , bool SO2 > // Storage order of the right-hand side matrix
1473inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO2>& rhs )
1474 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1475{
1479
1480 using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
1481
1484
1485 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
1486 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1487 }
1488
1489 if( !tryAddAssign( matrix_, *rhs, row(), column() ) ) {
1490 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1491 }
1492
1493 decltype(auto) left( derestrict( *this ) );
1494
1495 if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) || (*rhs).canAlias( this ) ) {
1496 const AddType tmp( *this + (*rhs) );
1497 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
1498 }
1499 else {
1500 smpAddAssign( left, transIf< IsSymmetric_v<This> >( *rhs ) );
1501 }
1502
1503 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1504
1505 return *this;
1506}
1508//*************************************************************************************************
1509
1510
1511//*************************************************************************************************
1525template< typename MT // Type of the dense matrix
1526 , size_t... CSAs > // Compile time submatrix arguments
1527template< typename MT2 // Type of the right-hand side matrix
1528 , bool SO2 > // Storage order of the right-hand side matrix
1529inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO2>& rhs )
1530 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1531{
1535
1536 using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
1537
1540
1541 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
1542 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1543 }
1544
1545 const AddType tmp( *this + (*rhs) );
1546
1547 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1548 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1549 }
1550
1551 decltype(auto) left( derestrict( *this ) );
1552
1553 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
1554
1555 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1556
1557 return *this;
1558}
1560//*************************************************************************************************
1561
1562
1563//*************************************************************************************************
1577template< typename MT // Type of the dense matrix
1578 , size_t... CSAs > // Compile time submatrix arguments
1579template< typename MT2 // Type of the right-hand side matrix
1580 , bool SO2 > // Storage order of the right-hand side matrix
1581inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO2>& rhs )
1582 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1583{
1587
1588 using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
1589
1592
1593 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
1594 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1595 }
1596
1597 if( !trySubAssign( matrix_, *rhs, row(), column() ) ) {
1598 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1599 }
1600
1601 decltype(auto) left( derestrict( *this ) );
1602
1603 if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) || (*rhs).canAlias( this ) ) {
1604 const SubType tmp( *this - (*rhs ) );
1605 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
1606 }
1607 else {
1608 smpSubAssign( left, transIf< IsSymmetric_v<This> >( *rhs ) );
1609 }
1610
1611 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1612
1613 return *this;
1614}
1616//*************************************************************************************************
1617
1618
1619//*************************************************************************************************
1633template< typename MT // Type of the dense matrix
1634 , size_t... CSAs > // Compile time submatrix arguments
1635template< typename MT2 // Type of the right-hand side matrix
1636 , bool SO2 > // Storage order of the right-hand side matrix
1637inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO2>& rhs )
1638 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1639{
1643
1644 using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
1645
1648
1649 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
1650 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1651 }
1652
1653 const SubType tmp( *this - (*rhs) );
1654
1655 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1656 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1657 }
1658
1659 decltype(auto) left( derestrict( *this ) );
1660
1661 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
1662
1663 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1664
1665 return *this;
1666}
1668//*************************************************************************************************
1669
1670
1671//*************************************************************************************************
1685template< typename MT // Type of the dense matrix
1686 , size_t... CSAs > // Compile time submatrix arguments
1687template< typename MT2 // Type of the right-hand side matrix
1688 , bool SO2 > // Storage order of the right-hand side matrix
1689inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO2>& rhs )
1690 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1691{
1695
1696 using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
1697
1699
1700 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
1701 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1702 }
1703
1704 if( !trySchurAssign( matrix_, *rhs, row(), column() ) ) {
1705 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1706 }
1707
1708 decltype(auto) left( derestrict( *this ) );
1709
1710 if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) || (*rhs).canAlias( this ) ) {
1711 const SchurType tmp( *this % (*rhs) );
1712 if( IsSparseMatrix_v<SchurType> )
1713 reset();
1714 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
1715 }
1716 else {
1717 smpSchurAssign( left, transIf< IsSymmetric_v<This> >( *rhs ) );
1718 }
1719
1720 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1721
1722 return *this;
1723}
1725//*************************************************************************************************
1726
1727
1728//*************************************************************************************************
1742template< typename MT // Type of the dense matrix
1743 , size_t... CSAs > // Compile time submatrix arguments
1744template< typename MT2 // Type of the right-hand side matrix
1745 , bool SO2 > // Storage order of the right-hand side matrix
1746inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO2>& rhs )
1747 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1748{
1752
1753 using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
1754
1756
1757 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
1758 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1759 }
1760
1761 const SchurType tmp( *this % (*rhs) );
1762
1763 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1764 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1765 }
1766
1767 decltype(auto) left( derestrict( *this ) );
1768
1769 if( IsSparseMatrix_v<SchurType> ) {
1770 reset();
1771 }
1772
1773 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
1774
1775 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1776
1777 return *this;
1778}
1780//*************************************************************************************************
1781
1782
1783
1784
1785//=================================================================================================
1786//
1787// UTILITY FUNCTIONS
1788//
1789//=================================================================================================
1790
1791//*************************************************************************************************
1797template< typename MT // Type of the dense matrix
1798 , size_t... CSAs > // Compile time submatrix arguments
1799inline MT& Submatrix<MT,unaligned,false,true,CSAs...>::operand() noexcept
1800{
1801 return matrix_;
1802}
1804//*************************************************************************************************
1805
1806
1807//*************************************************************************************************
1813template< typename MT // Type of the dense matrix
1814 , size_t... CSAs > // Compile time submatrix arguments
1815inline const MT& Submatrix<MT,unaligned,false,true,CSAs...>::operand() const noexcept
1816{
1817 return matrix_;
1818}
1820//*************************************************************************************************
1821
1822
1823//*************************************************************************************************
1834template< typename MT // Type of the dense matrix
1835 , size_t... CSAs > // Compile time submatrix arguments
1836inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::spacing() const noexcept
1837{
1838 return matrix_.spacing();
1839}
1841//*************************************************************************************************
1842
1843
1844//*************************************************************************************************
1850template< typename MT // Type of the dense matrix
1851 , size_t... CSAs > // Compile time submatrix arguments
1852inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::capacity() const noexcept
1853{
1854 return rows() * columns();
1855}
1857//*************************************************************************************************
1858
1859
1860//*************************************************************************************************
1872template< typename MT // Type of the dense matrix
1873 , size_t... CSAs > // Compile time submatrix arguments
1874inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::capacity( size_t i ) const noexcept
1875{
1876 MAYBE_UNUSED( i );
1877
1878 BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1879
1880 return columns();
1881}
1883//*************************************************************************************************
1884
1885
1886//*************************************************************************************************
1892template< typename MT // Type of the dense matrix
1893 , size_t... CSAs > // Compile time submatrix arguments
1895{
1896 const size_t iend( row() + rows() );
1897 const size_t jend( column() + columns() );
1898 size_t nonzeros( 0UL );
1899
1900 for( size_t i=row(); i<iend; ++i )
1901 for( size_t j=column(); j<jend; ++j )
1902 if( !isDefault( matrix_(i,j) ) )
1903 ++nonzeros;
1904
1905 return nonzeros;
1906}
1908//*************************************************************************************************
1909
1910
1911//*************************************************************************************************
1923template< typename MT // Type of the dense matrix
1924 , size_t... CSAs > // Compile time submatrix arguments
1925inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::nonZeros( size_t i ) const
1926{
1927 BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1928
1929 const size_t jend( column() + columns() );
1930 size_t nonzeros( 0UL );
1931
1932 for( size_t j=column(); j<jend; ++j )
1933 if( !isDefault( matrix_(row()+i,j) ) )
1934 ++nonzeros;
1935
1936 return nonzeros;
1937}
1939//*************************************************************************************************
1940
1941
1942//*************************************************************************************************
1948template< typename MT // Type of the dense matrix
1949 , size_t... CSAs > // Compile time submatrix arguments
1951{
1952 using blaze::clear;
1953
1954 for( size_t i=row(); i<row()+rows(); ++i )
1955 {
1956 const size_t jbegin( ( IsUpper_v<MT> )
1957 ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
1958 ?( max( i+1UL, column() ) )
1959 :( max( i, column() ) ) )
1960 :( column() ) );
1961 const size_t jend ( ( IsLower_v<MT> )
1962 ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
1963 ?( min( i, column()+columns() ) )
1964 :( min( i+1UL, column()+columns() ) ) )
1965 :( column()+columns() ) );
1966
1967 for( size_t j=jbegin; j<jend; ++j )
1968 clear( matrix_(i,j) );
1969 }
1970}
1972//*************************************************************************************************
1973
1974
1975//*************************************************************************************************
1987template< typename MT // Type of the dense matrix
1988 , size_t... CSAs > // Compile time submatrix arguments
1990{
1991 using blaze::clear;
1992
1993 BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1994
1995 const size_t jbegin( ( IsUpper_v<MT> )
1996 ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
1997 ?( max( i+1UL, column() ) )
1998 :( max( i, column() ) ) )
1999 :( column() ) );
2000 const size_t jend ( ( IsLower_v<MT> )
2001 ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
2002 ?( min( i, column()+columns() ) )
2003 :( min( i+1UL, column()+columns() ) ) )
2004 :( column()+columns() ) );
2005
2006 for( size_t j=jbegin; j<jend; ++j )
2007 clear( matrix_(row()+i,j) );
2008}
2010//*************************************************************************************************
2011
2012
2013//*************************************************************************************************
2023template< typename MT // Type of the dense matrix
2024 , size_t... CSAs > // Compile time submatrix arguments
2025inline bool Submatrix<MT,unaligned,false,true,CSAs...>::hasOverlap() const noexcept
2026{
2027 BLAZE_INTERNAL_ASSERT( IsSymmetric_v<MT> || IsHermitian_v<MT>, "Invalid matrix detected" );
2028
2029 if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
2030 return false;
2031 else return true;
2032}
2034//*************************************************************************************************
2035
2036
2037
2038
2039//=================================================================================================
2040//
2041// NUMERIC FUNCTIONS
2042//
2043//=================================================================================================
2044
2045//*************************************************************************************************
2063template< typename MT // Type of the dense matrix
2064 , size_t... CSAs > // Compile time submatrix arguments
2065inline Submatrix<MT,unaligned,false,true,CSAs...>&
2067{
2068 if( rows() != columns() ) {
2069 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
2070 }
2071
2072 if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
2073 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
2074 }
2075
2076 decltype(auto) left( derestrict( *this ) );
2077 const ResultType tmp( trans( *this ) );
2078
2079 smpAssign( left, tmp );
2080
2081 return *this;
2082}
2084//*************************************************************************************************
2085
2086
2087//*************************************************************************************************
2105template< typename MT // Type of the dense matrix
2106 , size_t... CSAs > // Compile time submatrix arguments
2107inline Submatrix<MT,unaligned,false,true,CSAs...>&
2109{
2110 if( rows() != columns() ) {
2111 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
2112 }
2113
2114 if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
2115 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
2116 }
2117
2118 decltype(auto) left( derestrict( *this ) );
2119 const ResultType tmp( ctrans( *this ) );
2120
2121 smpAssign( left, tmp );
2122
2123 return *this;
2124}
2126//*************************************************************************************************
2127
2128
2129//*************************************************************************************************
2142template< typename MT // Type of the dense matrix
2143 , size_t... CSAs > // Compile time submatrix arguments
2144template< typename Other > // Data type of the scalar value
2145inline Submatrix<MT,unaligned,false,true,CSAs...>&
2146 Submatrix<MT,unaligned,false,true,CSAs...>::scale( const Other& scalar )
2147{
2149
2150 const size_t iend( row() + rows() );
2151
2152 for( size_t i=row(); i<iend; ++i )
2153 {
2154 const size_t jbegin( ( IsUpper_v<MT> )
2155 ?( ( IsStrictlyUpper_v<MT> )
2156 ?( max( i+1UL, column() ) )
2157 :( max( i, column() ) ) )
2158 :( column() ) );
2159 const size_t jend ( ( IsLower_v<MT> )
2160 ?( ( IsStrictlyLower_v<MT> )
2161 ?( min( i, column()+columns() ) )
2162 :( min( i+1UL, column()+columns() ) ) )
2163 :( column()+columns() ) );
2164
2165 for( size_t j=jbegin; j<jend; ++j )
2166 matrix_(i,j) *= scalar;
2167 }
2168
2169 return *this;
2170}
2172//*************************************************************************************************
2173
2174
2175
2176
2177//=================================================================================================
2178//
2179// EXPRESSION TEMPLATE EVALUATION FUNCTIONS
2180//
2181//=================================================================================================
2182
2183//*************************************************************************************************
2194template< typename MT // Type of the dense matrix
2195 , size_t... CSAs > // Compile time submatrix arguments
2196template< typename Other > // Data type of the foreign expression
2197inline bool Submatrix<MT,unaligned,false,true,CSAs...>::canAlias( const Other* alias ) const noexcept
2198{
2199 return matrix_.isAliased( &unview( *alias ) );
2200}
2202//*************************************************************************************************
2203
2204
2205//*************************************************************************************************
2216template< typename MT // Type of the dense matrix
2217 , size_t... CSAs > // Compile time submatrix arguments
2218template< typename MT2 // Data type of the foreign dense submatrix
2219 , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
2220 , bool SO2 // Storage order of the foreign dense submatrix
2221 , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
2222inline bool
2223 Submatrix<MT,unaligned,false,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
2224{
2225 return ( matrix_.isAliased( &alias->matrix_ ) &&
2226 ( row() + rows() > alias->row() ) &&
2227 ( row() < alias->row() + alias->rows() ) &&
2228 ( column() + columns() > alias->column() ) &&
2229 ( column() < alias->column() + alias->columns() ) );
2230}
2232//*************************************************************************************************
2233
2234
2235//*************************************************************************************************
2246template< typename MT // Type of the dense matrix
2247 , size_t... CSAs > // Compile time submatrix arguments
2248template< typename Other > // Data type of the foreign expression
2249inline bool Submatrix<MT,unaligned,false,true,CSAs...>::isAliased( const Other* alias ) const noexcept
2250{
2251 return matrix_.isAliased( &unview( *alias ) );
2252}
2254//*************************************************************************************************
2255
2256
2257//*************************************************************************************************
2268template< typename MT // Type of the dense matrix
2269 , size_t... CSAs > // Compile time submatrix arguments
2270template< typename MT2 // Data type of the foreign dense submatrix
2271 , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
2272 , bool SO2 // Storage order of the foreign dense submatrix
2273 , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
2274inline bool
2275 Submatrix<MT,unaligned,false,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
2276{
2277 return ( matrix_.isAliased( &alias->matrix_ ) &&
2278 ( row() + rows() > alias->row() ) &&
2279 ( row() < alias->row() + alias->rows() ) &&
2280 ( column() + columns() > alias->column() ) &&
2281 ( column() < alias->column() + alias->columns() ) );
2282}
2284//*************************************************************************************************
2285
2286
2287//*************************************************************************************************
2297template< typename MT // Type of the dense matrix
2298 , size_t... CSAs > // Compile time submatrix arguments
2299inline bool Submatrix<MT,unaligned,false,true,CSAs...>::isAligned() const noexcept
2300{
2301 return isAligned_;
2302}
2304//*************************************************************************************************
2305
2306
2307//*************************************************************************************************
2318template< typename MT // Type of the dense matrix
2319 , size_t... CSAs > // Compile time submatrix arguments
2320inline bool Submatrix<MT,unaligned,false,true,CSAs...>::canSMPAssign() const noexcept
2321{
2322 return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
2323}
2325//*************************************************************************************************
2326
2327
2328//*************************************************************************************************
2344template< typename MT // Type of the dense matrix
2345 , size_t... CSAs > // Compile time submatrix arguments
2346BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,false,true,CSAs...>::SIMDType
2347 Submatrix<MT,unaligned,false,true,CSAs...>::load( size_t i, size_t j ) const noexcept
2348{
2349 if( isAligned_ )
2350 return loada( i, j );
2351 else
2352 return loadu( i, j );
2353}
2355//*************************************************************************************************
2356
2357
2358//*************************************************************************************************
2374template< typename MT // Type of the dense matrix
2375 , size_t... CSAs > // Compile time submatrix arguments
2376BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,false,true,CSAs...>::SIMDType
2377 Submatrix<MT,unaligned,false,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
2378{
2380
2381 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2382 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2383 BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2384 BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2385
2386 return matrix_.loada( row()+i, column()+j );
2387}
2389//*************************************************************************************************
2390
2391
2392//*************************************************************************************************
2408template< typename MT // Type of the dense matrix
2409 , size_t... CSAs > // Compile time submatrix arguments
2410BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,false,true,CSAs...>::SIMDType
2411 Submatrix<MT,unaligned,false,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
2412{
2414
2415 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2416 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2417 BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2418
2419 return matrix_.loadu( row()+i, column()+j );
2420}
2422//*************************************************************************************************
2423
2424
2425//*************************************************************************************************
2442template< typename MT // Type of the dense matrix
2443 , size_t... CSAs > // Compile time submatrix arguments
2445 Submatrix<MT,unaligned,false,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
2446{
2447 if( isAligned_ )
2448 storea( i, j, value );
2449 else
2450 storeu( i, j, value );
2451}
2453//*************************************************************************************************
2454
2455
2456//*************************************************************************************************
2473template< typename MT // Type of the dense matrix
2474 , size_t... CSAs > // Compile time submatrix arguments
2476 Submatrix<MT,unaligned,false,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
2477{
2479
2480 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2481 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2482 BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2483 BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2484
2485 matrix_.storea( row()+i, column()+j, value );
2486}
2488//*************************************************************************************************
2489
2490
2491//*************************************************************************************************
2508template< typename MT // Type of the dense matrix
2509 , size_t... CSAs > // Compile time submatrix arguments
2511 Submatrix<MT,unaligned,false,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
2512{
2514
2515 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2516 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2517 BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2518
2519 matrix_.storeu( row()+i, column()+j, value );
2520}
2522//*************************************************************************************************
2523
2524
2525//*************************************************************************************************
2542template< typename MT // Type of the dense matrix
2543 , size_t... CSAs > // Compile time submatrix arguments
2545 Submatrix<MT,unaligned,false,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
2546{
2548
2549 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2550 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2551 BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2552 BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2553
2554 if( isAligned_ )
2555 matrix_.stream( row()+i, column()+j, value );
2556 else
2557 matrix_.storeu( row()+i, column()+j, value );
2558}
2560//*************************************************************************************************
2561
2562
2563//*************************************************************************************************
2575template< typename MT // Type of the dense matrix
2576 , size_t... CSAs > // Compile time submatrix arguments
2577template< typename MT2 > // Type of the right-hand side dense matrix
2578inline auto Submatrix<MT,unaligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
2579 -> DisableIf_t< VectorizedAssign_v<MT2> >
2580{
2581 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
2582 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
2583
2584 const size_t jpos( prevMultiple( columns(), 2UL ) );
2585 BLAZE_INTERNAL_ASSERT( jpos <= columns(), "Invalid end calculation" );
2586
2587 for( size_t i=0UL; i<rows(); ++i ) {
2588 for( size_t j=0UL; j<jpos; j+=2UL ) {
2589 matrix_(row()+i,column()+j ) = (*rhs)(i,j );
2590 matrix_(row()+i,column()+j+1UL) = (*rhs)(i,j+1UL);
2591 }
2592 if( jpos < columns() ) {
2593 matrix_(row()+i,column()+jpos) = (*rhs)(i,jpos);
2594 }
2595 }
2596}
2598//*************************************************************************************************
2599
2600
2601//*************************************************************************************************
2613template< typename MT // Type of the dense matrix
2614 , size_t... CSAs > // Compile time submatrix arguments
2615template< typename MT2 > // Type of the right-hand side dense matrix
2616inline auto Submatrix<MT,unaligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
2617 -> EnableIf_t< VectorizedAssign_v<MT2> >
2618{
2620
2621 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
2622 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
2623
2624 const size_t jpos( prevMultiple( columns(), SIMDSIZE ) );
2625 BLAZE_INTERNAL_ASSERT( jpos <= columns(), "Invalid end calculation" );
2626
2627 if( useStreaming && isAligned_ &&
2628 rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
2629 !(*rhs).isAliased( this ) )
2630 {
2631 for( size_t i=0UL; i<rows(); ++i )
2632 {
2633 size_t j( 0UL );
2634 Iterator left( begin(i) );
2635 ConstIterator_t<MT2> right( (*rhs).begin(i) );
2636
2637 for( ; j<jpos; j+=SIMDSIZE ) {
2638 left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2639 }
2640 for( ; j<columns(); ++j ) {
2641 *left = *right; ++left; ++right;
2642 }
2643 }
2644 }
2645 else
2646 {
2647 for( size_t i=0UL; i<rows(); ++i )
2648 {
2649 size_t j( 0UL );
2650 Iterator left( begin(i) );
2651 ConstIterator_t<MT2> right( (*rhs).begin(i) );
2652
2653 for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
2654 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2655 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2656 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2657 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2658 }
2659 for( ; j<jpos; j+=SIMDSIZE ) {
2660 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2661 }
2662 for( ; j<columns(); ++j ) {
2663 *left = *right; ++left; ++right;
2664 }
2665 }
2666 }
2667}
2669//*************************************************************************************************
2670
2671
2672//*************************************************************************************************
2684template< typename MT // Type of the dense matrix
2685 , size_t... CSAs > // Compile time submatrix arguments
2686template< typename MT2 > // Type of the right-hand side dense matrix
2687inline void Submatrix<MT,unaligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
2688{
2690
2691 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
2692 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
2693
2694 constexpr size_t block( BLOCK_SIZE );
2695
2696 for( size_t ii=0UL; ii<rows(); ii+=block ) {
2697 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
2698 for( size_t jj=0UL; jj<columns(); jj+=block ) {
2699 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
2700 for( size_t i=ii; i<iend; ++i ) {
2701 for( size_t j=jj; j<jend; ++j ) {
2702 matrix_(row()+i,column()+j) = (*rhs)(i,j);
2703 }
2704 }
2705 }
2706 }
2707}
2709//*************************************************************************************************
2710
2711
2712//*************************************************************************************************
2724template< typename MT // Type of the dense matrix
2725 , size_t... CSAs > // Compile time submatrix arguments
2726template< typename MT2 > // Type of the right-hand side sparse matrix
2727inline void Submatrix<MT,unaligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
2728{
2729 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
2730 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
2731
2732 for( size_t i=0UL; i<rows(); ++i )
2733 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element )
2734 matrix_(row()+i,column()+element->index()) = element->value();
2735}
2737//*************************************************************************************************
2738
2739
2740//*************************************************************************************************
2752template< typename MT // Type of the dense matrix
2753 , size_t... CSAs > // Compile time submatrix arguments
2754template< typename MT2 > // Type of the right-hand side sparse matrix
2755inline void Submatrix<MT,unaligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
2756{
2758
2759 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
2760 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
2761
2762 for( size_t j=0UL; j<columns(); ++j )
2763 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element )
2764 matrix_(row()+element->index(),column()+j) = element->value();
2765}
2767//*************************************************************************************************
2768
2769
2770//*************************************************************************************************
2782template< typename MT // Type of the dense matrix
2783 , size_t... CSAs > // Compile time submatrix arguments
2784template< typename MT2 > // Type of the right-hand side dense matrix
2785inline auto Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
2786 -> DisableIf_t< VectorizedAddAssign_v<MT2> >
2787{
2788 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
2789 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
2790
2791 const size_t jpos( prevMultiple( columns(), 2UL ) );
2792 BLAZE_INTERNAL_ASSERT( jpos <= columns(), "Invalid end calculation" );
2793
2794 for( size_t i=0UL; i<rows(); ++i )
2795 {
2796 if( IsDiagonal_v<MT2> ) {
2797 matrix_(row()+i,column()+i) += (*rhs)(i,i);
2798 }
2799 else {
2800 for( size_t j=0UL; j<jpos; j+=2UL ) {
2801 matrix_(row()+i,column()+j ) += (*rhs)(i,j );
2802 matrix_(row()+i,column()+j+1UL) += (*rhs)(i,j+1UL);
2803 }
2804 if( jpos < columns() ) {
2805 matrix_(row()+i,column()+jpos) += (*rhs)(i,jpos);
2806 }
2807 }
2808 }
2809}
2811//*************************************************************************************************
2812
2813
2814//*************************************************************************************************
2826template< typename MT // Type of the dense matrix
2827 , size_t... CSAs > // Compile time submatrix arguments
2828template< typename MT2 > // Type of the right-hand side dense matrix
2829inline auto Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
2830 -> EnableIf_t< VectorizedAddAssign_v<MT2> >
2831{
2833
2834 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
2835 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
2836
2837 for( size_t i=0UL; i<rows(); ++i )
2838 {
2839 const size_t jbegin( ( IsUpper_v<MT2> )
2840 ?( prevMultiple( ( IsStrictlyUpper_v<MT2> ? i+1UL : i ), SIMDSIZE ) )
2841 :( 0UL ) );
2842 const size_t jend ( ( IsLower_v<MT2> )
2843 ?( IsStrictlyLower_v<MT2> ? i : i+1UL )
2844 :( columns() ) );
2845 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
2846
2847 const size_t jpos( prevMultiple( jend, SIMDSIZE ) );
2848 BLAZE_INTERNAL_ASSERT( jpos <= jend, "Invalid end calculation" );
2849
2850 size_t j( jbegin );
2851 Iterator left( begin(i) + jbegin );
2852 ConstIterator_t<MT2> right( (*rhs).begin(i) + jbegin );
2853
2854 for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
2855 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2856 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2857 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2858 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2859 }
2860 for( ; j<jpos; j+=SIMDSIZE ) {
2861 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2862 }
2863 for( ; j<jend; ++j ) {
2864 *left += *right; ++left; ++right;
2865 }
2866 }
2867}
2869//*************************************************************************************************
2870
2871
2872//*************************************************************************************************
2884template< typename MT // Type of the dense matrix
2885 , size_t... CSAs > // Compile time submatrix arguments
2886template< typename MT2 > // Type of the right-hand side dense matrix
2887inline void Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
2888{
2890
2891 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
2892 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
2893
2894 constexpr size_t block( BLOCK_SIZE );
2895
2896 for( size_t ii=0UL; ii<rows(); ii+=block ) {
2897 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
2898 for( size_t jj=0UL; jj<columns(); jj+=block ) {
2899 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
2900 for( size_t i=ii; i<iend; ++i ) {
2901 for( size_t j=jj; j<jend; ++j ) {
2902 matrix_(row()+i,column()+j) += (*rhs)(i,j);
2903 }
2904 }
2905 }
2906 }
2907}
2909//*************************************************************************************************
2910
2911
2912//*************************************************************************************************
2924template< typename MT // Type of the dense matrix
2925 , size_t... CSAs > // Compile time submatrix arguments
2926template< typename MT2 > // Type of the right-hand side sparse matrix
2927inline void Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
2928{
2929 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
2930 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
2931
2932 for( size_t i=0UL; i<rows(); ++i )
2933 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element )
2934 matrix_(row()+i,column()+element->index()) += element->value();
2935}
2937//*************************************************************************************************
2938
2939
2940//*************************************************************************************************
2952template< typename MT // Type of the dense matrix
2953 , size_t... CSAs > // Compile time submatrix arguments
2954template< typename MT2 > // Type of the right-hand side sparse matrix
2955inline void Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
2956{
2958
2959 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
2960 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
2961
2962 for( size_t j=0UL; j<columns(); ++j )
2963 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element )
2964 matrix_(row()+element->index(),column()+j) += element->value();
2965}
2967//*************************************************************************************************
2968
2969
2970//*************************************************************************************************
2982template< typename MT // Type of the dense matrix
2983 , size_t... CSAs > // Compile time submatrix arguments
2984template< typename MT2 > // Type of the right-hand side dense matrix
2985inline auto Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
2986 -> DisableIf_t< VectorizedSubAssign_v<MT2> >
2987{
2988 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
2989 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
2990
2991 const size_t jpos( prevMultiple( columns(), 2UL ) );
2992 BLAZE_INTERNAL_ASSERT( jpos <= columns(), "Invalid end calculation" );
2993
2994 for( size_t i=0UL; i<rows(); ++i )
2995 {
2996 if( IsDiagonal_v<MT2> ) {
2997 matrix_(row()+i,column()+i) -= (*rhs)(i,i);
2998 }
2999 else {
3000 for( size_t j=0UL; j<jpos; j+=2UL ) {
3001 matrix_(row()+i,column()+j ) -= (*rhs)(i,j );
3002 matrix_(row()+i,column()+j+1UL) -= (*rhs)(i,j+1UL);
3003 }
3004 if( jpos < columns() ) {
3005 matrix_(row()+i,column()+jpos) -= (*rhs)(i,jpos);
3006 }
3007 }
3008 }
3009}
3011//*************************************************************************************************
3012
3013
3014//*************************************************************************************************
3026template< typename MT // Type of the dense matrix
3027 , size_t... CSAs > // Compile time submatrix arguments
3028template< typename MT2 > // Type of the right-hand side dense matrix
3029inline auto Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
3030 -> EnableIf_t< VectorizedSubAssign_v<MT2> >
3031{
3033
3034 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
3035 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
3036
3037 for( size_t i=0UL; i<rows(); ++i )
3038 {
3039 const size_t jbegin( ( IsUpper_v<MT2> )
3040 ?( prevMultiple( ( IsStrictlyUpper_v<MT2> ? i+1UL : i ), SIMDSIZE ) )
3041 :( 0UL ) );
3042 const size_t jend ( ( IsLower_v<MT2> )
3043 ?( IsStrictlyLower_v<MT2> ? i : i+1UL )
3044 :( columns() ) );
3045 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
3046
3047 const size_t jpos( prevMultiple( jend, SIMDSIZE ) );
3048 BLAZE_INTERNAL_ASSERT( jpos <= jend, "Invalid end calculation" );
3049
3050 size_t j( jbegin );
3051 Iterator left( begin(i) + jbegin );
3052 ConstIterator_t<MT2> right( (*rhs).begin(i) + jbegin );
3053
3054 for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
3055 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3056 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3057 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3058 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3059 }
3060 for( ; j<jpos; j+=SIMDSIZE ) {
3061 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3062 }
3063 for( ; j<jend; ++j ) {
3064 *left -= *right; ++left; ++right;
3065 }
3066 }
3067}
3069//*************************************************************************************************
3070
3071
3072//*************************************************************************************************
3084template< typename MT // Type of the dense matrix
3085 , size_t... CSAs > // Compile time submatrix arguments
3086template< typename MT2 > // Type of the right-hand side dense matrix
3087inline void Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
3088{
3090
3091 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
3092 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
3093
3094 constexpr size_t block( BLOCK_SIZE );
3095
3096 for( size_t ii=0UL; ii<rows(); ii+=block ) {
3097 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
3098 for( size_t jj=0UL; jj<columns(); jj+=block ) {
3099 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
3100 for( size_t i=ii; i<iend; ++i ) {
3101 for( size_t j=jj; j<jend; ++j ) {
3102 matrix_(row()+i,column()+j) -= (*rhs)(i,j);
3103 }
3104 }
3105 }
3106 }
3107}
3109//*************************************************************************************************
3110
3111
3112//*************************************************************************************************
3124template< typename MT // Type of the dense matrix
3125 , size_t... CSAs > // Compile time submatrix arguments
3126template< typename MT2 > // Type of the right-hand side sparse matrix
3127inline void Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
3128{
3129 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
3130 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
3131
3132 for( size_t i=0UL; i<rows(); ++i )
3133 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element )
3134 matrix_(row()+i,column()+element->index()) -= element->value();
3135}
3137//*************************************************************************************************
3138
3139
3140//*************************************************************************************************
3152template< typename MT // Type of the dense matrix
3153 , size_t... CSAs > // Compile time submatrix arguments
3154template< typename MT2 > // Type of the right-hand side sparse matrix
3155inline void Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
3156{
3158
3159 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
3160 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
3161
3162 for( size_t j=0UL; j<columns(); ++j )
3163 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element )
3164 matrix_(row()+element->index(),column()+j) -= element->value();
3165}
3167//*************************************************************************************************
3168
3169
3170//*************************************************************************************************
3182template< typename MT // Type of the dense matrix
3183 , size_t... CSAs > // Compile time submatrix arguments
3184template< typename MT2 > // Type of the right-hand side dense matrix
3185inline auto Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
3186 -> DisableIf_t< VectorizedSchurAssign_v<MT2> >
3187{
3188 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
3189 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
3190
3191 const size_t jpos( prevMultiple( columns(), 2UL ) );
3192 BLAZE_INTERNAL_ASSERT( jpos <= columns(), "Invalid end calculation" );
3193
3194 for( size_t i=0UL; i<rows(); ++i ) {
3195 for( size_t j=0UL; j<jpos; j+=2UL ) {
3196 matrix_(row()+i,column()+j ) *= (*rhs)(i,j );
3197 matrix_(row()+i,column()+j+1UL) *= (*rhs)(i,j+1UL);
3198 }
3199 if( jpos < columns() ) {
3200 matrix_(row()+i,column()+jpos) *= (*rhs)(i,jpos);
3201 }
3202 }
3203}
3205//*************************************************************************************************
3206
3207
3208//*************************************************************************************************
3220template< typename MT // Type of the dense matrix
3221 , size_t... CSAs > // Compile time submatrix arguments
3222template< typename MT2 > // Type of the right-hand side dense matrix
3223inline auto Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
3224 -> EnableIf_t< VectorizedSchurAssign_v<MT2> >
3225{
3227
3228 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
3229 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
3230
3231 for( size_t i=0UL; i<rows(); ++i )
3232 {
3233 const size_t jpos( prevMultiple( columns(), SIMDSIZE ) );
3234 BLAZE_INTERNAL_ASSERT( jpos <= columns(), "Invalid end calculation" );
3235
3236 size_t j( 0UL );
3237 Iterator left( begin(i) );
3238 ConstIterator_t<MT2> right( (*rhs).begin(i) );
3239
3240 for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
3241 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3242 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3243 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3244 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3245 }
3246 for( ; j<jpos; j+=SIMDSIZE ) {
3247 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3248 }
3249 for( ; j<columns(); ++j ) {
3250 *left *= *right; ++left; ++right;
3251 }
3252 }
3253}
3255//*************************************************************************************************
3256
3257
3258//*************************************************************************************************
3270template< typename MT // Type of the dense matrix
3271 , size_t... CSAs > // Compile time submatrix arguments
3272template< typename MT2 > // Type of the right-hand side dense matrix
3273inline void Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
3274{
3276
3277 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
3278 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
3279
3280 constexpr size_t block( BLOCK_SIZE );
3281
3282 for( size_t ii=0UL; ii<rows(); ii+=block ) {
3283 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
3284 for( size_t jj=0UL; jj<columns(); jj+=block ) {
3285 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
3286 for( size_t i=ii; i<iend; ++i ) {
3287 for( size_t j=jj; j<jend; ++j ) {
3288 matrix_(row()+i,column()+j) *= (*rhs)(i,j);
3289 }
3290 }
3291 }
3292 }
3293}
3295//*************************************************************************************************
3296
3297
3298//*************************************************************************************************
3310template< typename MT // Type of the dense matrix
3311 , size_t... CSAs > // Compile time submatrix arguments
3312template< typename MT2 > // Type of the right-hand side sparse matrix
3313inline void Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
3314{
3315 using blaze::reset;
3316
3317 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
3318 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
3319
3320 for( size_t i=0UL; i<rows(); ++i )
3321 {
3322 size_t j( 0UL );
3323
3324 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element ) {
3325 for( ; j<element->index(); ++j )
3326 reset( matrix_(row()+i,column()+j) );
3327 matrix_(row()+i,column()+j) *= element->value();
3328 ++j;
3329 }
3330
3331 for( ; j<columns(); ++j ) {
3332 reset( matrix_(row()+i,column()+j) );
3333 }
3334 }
3335}
3337//*************************************************************************************************
3338
3339
3340//*************************************************************************************************
3352template< typename MT // Type of the dense matrix
3353 , size_t... CSAs > // Compile time submatrix arguments
3354template< typename MT2 > // Type of the right-hand side sparse matrix
3355inline void Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
3356{
3357 using blaze::reset;
3358
3360
3361 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
3362 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
3363
3364 for( size_t j=0UL; j<columns(); ++j )
3365 {
3366 size_t i( 0UL );
3367
3368 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element ) {
3369 for( ; i<element->index(); ++i )
3370 reset( matrix_(row()+i,column()+j) );
3371 matrix_(row()+i,column()+j) *= element->value();
3372 ++i;
3373 }
3374
3375 for( ; i<rows(); ++i ) {
3376 reset( matrix_(row()+i,column()+j) );
3377 }
3378 }
3379}
3381//*************************************************************************************************
3382
3383
3384
3385
3386
3387
3388
3389
3390//=================================================================================================
3391//
3392// CLASS TEMPLATE SPECIALIZATION FOR UNALIGNED COLUMN-MAJOR DENSE SUBMATRICES
3393//
3394//=================================================================================================
3395
3396//*************************************************************************************************
3404template< typename MT // Type of the dense matrix
3405 , size_t... CSAs > // Compile time submatrix arguments
3406class Submatrix<MT,unaligned,true,true,CSAs...>
3407 : public View< DenseMatrix< Submatrix<MT,unaligned,true,true,CSAs...>, true > >
3408 , private SubmatrixData<CSAs...>
3409{
3410 private:
3411 //**Type definitions****************************************************************************
3412 using DataType = SubmatrixData<CSAs...>;
3413 using Operand = If_t< IsExpression_v<MT>, MT, MT& >;
3414 //**********************************************************************************************
3415
3416 //**********************************************************************************************
3418 template< typename MT1, typename MT2 >
3419 static constexpr bool EnforceEvaluation_v =
3420 ( IsRestricted_v<MT1> && RequiresEvaluation_v<MT2> );
3421 //**********************************************************************************************
3422
3423 public:
3424 //**Type definitions****************************************************************************
3426 using This = Submatrix<MT,unaligned,true,true,CSAs...>;
3427
3429 using BaseType = View< DenseMatrix<This,true> >;
3430
3431 using ViewedType = MT;
3432 using ResultType = SubmatrixTrait_t<MT,CSAs...>;
3433 using OppositeType = OppositeType_t<ResultType>;
3434 using TransposeType = TransposeType_t<ResultType>;
3435 using ElementType = ElementType_t<MT>;
3436 using SIMDType = SIMDTrait_t<ElementType>;
3437 using ReturnType = ReturnType_t<MT>;
3438 using CompositeType = const Submatrix&;
3439
3441 using ConstReference = ConstReference_t<MT>;
3442
3444 using Reference = If_t< IsConst_v<MT>, ConstReference, Reference_t<MT> >;
3445
3447 using ConstPointer = ConstPointer_t<MT>;
3448
3450 using Pointer = If_t< IsConst_v<MT> || !HasMutableDataAccess_v<MT>, ConstPointer, Pointer_t<MT> >;
3451 //**********************************************************************************************
3452
3453 //**SubmatrixIterator class definition**********************************************************
3456 template< typename IteratorType > // Type of the dense matrix iterator
3457 class SubmatrixIterator
3458 {
3459 public:
3460 //**Type definitions*************************************************************************
3462 using IteratorCategory = typename std::iterator_traits<IteratorType>::iterator_category;
3463
3465 using ValueType = typename std::iterator_traits<IteratorType>::value_type;
3466
3468 using PointerType = typename std::iterator_traits<IteratorType>::pointer;
3469
3471 using ReferenceType = typename std::iterator_traits<IteratorType>::reference;
3472
3474 using DifferenceType = typename std::iterator_traits<IteratorType>::difference_type;
3475
3476 // STL iterator requirements
3477 using iterator_category = IteratorCategory;
3478 using value_type = ValueType;
3479 using pointer = PointerType;
3480 using reference = ReferenceType;
3481 using difference_type = DifferenceType;
3482 //*******************************************************************************************
3483
3484 //**Constructor******************************************************************************
3487 inline SubmatrixIterator()
3488 : iterator_ ( ) // Iterator to the current submatrix element
3489 , isAligned_( false ) // Memory alignment flag
3490 {}
3491 //*******************************************************************************************
3492
3493 //**Constructor******************************************************************************
3501 inline SubmatrixIterator( IteratorType iterator, bool isMemoryAligned )
3502 : iterator_ ( iterator ) // Iterator to the current submatrix element
3503 , isAligned_( isMemoryAligned ) // Memory alignment flag
3504 {}
3505 //*******************************************************************************************
3506
3507 //**Constructor******************************************************************************
3512 template< typename IteratorType2 >
3513 inline SubmatrixIterator( const SubmatrixIterator<IteratorType2>& it )
3514 : iterator_ ( it.base() ) // Iterator to the current submatrix element
3515 , isAligned_( it.isAligned() ) // Memory alignment flag
3516 {}
3517 //*******************************************************************************************
3518
3519 //**Addition assignment operator*************************************************************
3525 inline SubmatrixIterator& operator+=( size_t inc ) {
3526 iterator_ += inc;
3527 return *this;
3528 }
3529 //*******************************************************************************************
3530
3531 //**Subtraction assignment operator**********************************************************
3537 inline SubmatrixIterator& operator-=( size_t dec ) {
3538 iterator_ -= dec;
3539 return *this;
3540 }
3541 //*******************************************************************************************
3542
3543 //**Prefix increment operator****************************************************************
3548 inline SubmatrixIterator& operator++() {
3549 ++iterator_;
3550 return *this;
3551 }
3552 //*******************************************************************************************
3553
3554 //**Postfix increment operator***************************************************************
3559 inline const SubmatrixIterator operator++( int ) {
3560 return SubmatrixIterator( iterator_++, isAligned_ );
3561 }
3562 //*******************************************************************************************
3563
3564 //**Prefix decrement operator****************************************************************
3569 inline SubmatrixIterator& operator--() {
3570 --iterator_;
3571 return *this;
3572 }
3573 //*******************************************************************************************
3574
3575 //**Postfix decrement operator***************************************************************
3580 inline const SubmatrixIterator operator--( int ) {
3581 return SubmatrixIterator( iterator_--, isAligned_ );
3582 }
3583 //*******************************************************************************************
3584
3585 //**Element access operator******************************************************************
3590 inline ReferenceType operator*() const {
3591 return *iterator_;
3592 }
3593 //*******************************************************************************************
3594
3595 //**Element access operator******************************************************************
3600 inline IteratorType operator->() const {
3601 return iterator_;
3602 }
3603 //*******************************************************************************************
3604
3605 //**Load function****************************************************************************
3615 inline SIMDType load() const noexcept {
3616 if( isAligned_ )
3617 return loada();
3618 else
3619 return loadu();
3620 }
3621 //*******************************************************************************************
3622
3623 //**Loada function***************************************************************************
3633 inline SIMDType loada() const noexcept {
3634 return iterator_.loada();
3635 }
3636 //*******************************************************************************************
3637
3638 //**Loadu function***************************************************************************
3648 inline SIMDType loadu() const noexcept {
3649 return iterator_.loadu();
3650 }
3651 //*******************************************************************************************
3652
3653 //**Store function***************************************************************************
3664 inline void store( const SIMDType& value ) const {
3665 if( isAligned_ ) {
3666 storea( value );
3667 }
3668 else {
3669 storeu( value );
3670 }
3671 }
3672 //*******************************************************************************************
3673
3674 //**Storea function**************************************************************************
3685 inline void storea( const SIMDType& value ) const {
3686 iterator_.storea( value );
3687 }
3688 //*******************************************************************************************
3689
3690 //**Storeu function**************************************************************************
3701 inline void storeu( const SIMDType& value ) const {
3702 iterator_.storeu( value );
3703 }
3704 //*******************************************************************************************
3705
3706 //**Stream function**************************************************************************
3717 inline void stream( const SIMDType& value ) const {
3718 iterator_.stream( value );
3719 }
3720 //*******************************************************************************************
3721
3722 //**Equality operator************************************************************************
3728 inline bool operator==( const SubmatrixIterator& rhs ) const {
3729 return iterator_ == rhs.iterator_;
3730 }
3731 //*******************************************************************************************
3732
3733 //**Inequality operator**********************************************************************
3739 inline bool operator!=( const SubmatrixIterator& rhs ) const {
3740 return iterator_ != rhs.iterator_;
3741 }
3742 //*******************************************************************************************
3743
3744 //**Less-than operator***********************************************************************
3750 inline bool operator<( const SubmatrixIterator& rhs ) const {
3751 return iterator_ < rhs.iterator_;
3752 }
3753 //*******************************************************************************************
3754
3755 //**Greater-than operator********************************************************************
3761 inline bool operator>( const SubmatrixIterator& rhs ) const {
3762 return iterator_ > rhs.iterator_;
3763 }
3764 //*******************************************************************************************
3765
3766 //**Less-or-equal-than operator**************************************************************
3772 inline bool operator<=( const SubmatrixIterator& rhs ) const {
3773 return iterator_ <= rhs.iterator_;
3774 }
3775 //*******************************************************************************************
3776
3777 //**Greater-or-equal-than operator***********************************************************
3783 inline bool operator>=( const SubmatrixIterator& rhs ) const {
3784 return iterator_ >= rhs.iterator_;
3785 }
3786 //*******************************************************************************************
3787
3788 //**Subtraction operator*********************************************************************
3794 inline DifferenceType operator-( const SubmatrixIterator& rhs ) const {
3795 return iterator_ - rhs.iterator_;
3796 }
3797 //*******************************************************************************************
3798
3799 //**Addition operator************************************************************************
3806 friend inline const SubmatrixIterator operator+( const SubmatrixIterator& it, size_t inc ) {
3807 return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
3808 }
3809 //*******************************************************************************************
3810
3811 //**Addition operator************************************************************************
3818 friend inline const SubmatrixIterator operator+( size_t inc, const SubmatrixIterator& it ) {
3819 return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
3820 }
3821 //*******************************************************************************************
3822
3823 //**Subtraction operator*********************************************************************
3830 friend inline const SubmatrixIterator operator-( const SubmatrixIterator& it, size_t dec ) {
3831 return SubmatrixIterator( it.iterator_ - dec, it.isAligned_ );
3832 }
3833 //*******************************************************************************************
3834
3835 //**Base function****************************************************************************
3840 inline IteratorType base() const {
3841 return iterator_;
3842 }
3843 //*******************************************************************************************
3844
3845 //**IsAligned function***********************************************************************
3850 inline bool isAligned() const noexcept {
3851 return isAligned_;
3852 }
3853 //*******************************************************************************************
3854
3855 private:
3856 //**Member variables*************************************************************************
3857 IteratorType iterator_;
3858 bool isAligned_;
3859 //*******************************************************************************************
3860 };
3861 //**********************************************************************************************
3862
3863 //**Type definitions****************************************************************************
3865 using ConstIterator = SubmatrixIterator< ConstIterator_t<MT> >;
3866
3868 using Iterator = If_t< IsConst_v<MT>, ConstIterator, SubmatrixIterator< Iterator_t<MT> > >;
3869 //**********************************************************************************************
3870
3871 //**Compilation flags***************************************************************************
3873 static constexpr bool simdEnabled = MT::simdEnabled;
3874
3876 static constexpr bool smpAssignable = MT::smpAssignable;
3877
3879 static constexpr bool compileTimeArgs = DataType::compileTimeArgs;
3880 //**********************************************************************************************
3881
3882 //**Constructors********************************************************************************
3885 template< typename... RSAs >
3886 explicit inline Submatrix( MT& matrix, RSAs... args );
3887
3888 Submatrix( const Submatrix& ) = default;
3890 //**********************************************************************************************
3891
3892 //**Destructor**********************************************************************************
3895 ~Submatrix() = default;
3897 //**********************************************************************************************
3898
3899 //**Data access functions***********************************************************************
3902 inline Reference operator()( size_t i, size_t j );
3903 inline ConstReference operator()( size_t i, size_t j ) const;
3904 inline Reference at( size_t i, size_t j );
3905 inline ConstReference at( size_t i, size_t j ) const;
3906 inline Pointer data () noexcept;
3907 inline ConstPointer data () const noexcept;
3908 inline Pointer data ( size_t j ) noexcept;
3909 inline ConstPointer data ( size_t j ) const noexcept;
3910 inline Iterator begin ( size_t j );
3911 inline ConstIterator begin ( size_t j ) const;
3912 inline ConstIterator cbegin( size_t j ) const;
3913 inline Iterator end ( size_t j );
3914 inline ConstIterator end ( size_t j ) const;
3915 inline ConstIterator cend ( size_t j ) const;
3917 //**********************************************************************************************
3918
3919 //**Assignment operators************************************************************************
3922 inline Submatrix& operator=( const ElementType& rhs );
3923 inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
3924 inline Submatrix& operator=( const Submatrix& rhs );
3925
3926 template< typename MT2, bool SO >
3927 inline Submatrix& operator=( const Matrix<MT2,SO>& rhs );
3928
3929 template< typename MT2, bool SO >
3930 inline auto operator+=( const Matrix<MT2,SO>& rhs )
3931 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3932
3933 template< typename MT2, bool SO >
3934 inline auto operator+=( const Matrix<MT2,SO>& rhs )
3935 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3936
3937 template< typename MT2, bool SO >
3938 inline auto operator-=( const Matrix<MT2,SO>& rhs )
3939 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3940
3941 template< typename MT2, bool SO >
3942 inline auto operator-=( const Matrix<MT2,SO>& rhs )
3943 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3944
3945 template< typename MT2, bool SO >
3946 inline auto operator%=( const Matrix<MT2,SO>& rhs )
3947 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3948
3949 template< typename MT2, bool SO >
3950 inline auto operator%=( const Matrix<MT2,SO>& rhs )
3951 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3953 //**********************************************************************************************
3954
3955 //**Utility functions***************************************************************************
3958 using DataType::row;
3959 using DataType::column;
3960 using DataType::rows;
3961 using DataType::columns;
3962
3963 inline MT& operand() noexcept;
3964 inline const MT& operand() const noexcept;
3965
3966 inline size_t spacing() const noexcept;
3967 inline size_t capacity() const noexcept;
3968 inline size_t capacity( size_t i ) const noexcept;
3969 inline size_t nonZeros() const;
3970 inline size_t nonZeros( size_t i ) const;
3971 inline void reset();
3972 inline void reset( size_t i );
3974 //**********************************************************************************************
3975
3976 //**Numeric functions***************************************************************************
3979 inline Submatrix& transpose();
3980 inline Submatrix& ctranspose();
3981
3982 template< typename Other > inline Submatrix& scale( const Other& scalar );
3984 //**********************************************************************************************
3985
3986 private:
3987 //**********************************************************************************************
3989 template< typename MT2 >
3990 static constexpr bool VectorizedAssign_v =
3991 ( useOptimizedKernels &&
3992 simdEnabled && MT2::simdEnabled &&
3993 IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > );
3994 //**********************************************************************************************
3995
3996 //**********************************************************************************************
3998 template< typename MT2 >
3999 static constexpr bool VectorizedAddAssign_v =
4000 ( VectorizedAssign_v<MT2> &&
4001 HasSIMDAdd_v< ElementType, ElementType_t<MT2> > &&
4002 !IsDiagonal_v<MT2> );
4003 //**********************************************************************************************
4004
4005 //**********************************************************************************************
4007 template< typename MT2 >
4008 static constexpr bool VectorizedSubAssign_v =
4009 ( VectorizedAssign_v<MT2> &&
4010 HasSIMDSub_v< ElementType, ElementType_t<MT2> > &&
4011 !IsDiagonal_v<MT2> );
4012 //**********************************************************************************************
4013
4014 //**********************************************************************************************
4016 template< typename MT2 >
4017 static constexpr bool VectorizedSchurAssign_v =
4018 ( VectorizedAssign_v<MT2> &&
4019 HasSIMDMult_v< ElementType, ElementType_t<MT2> > );
4020 //**********************************************************************************************
4021
4022 //**SIMD properties*****************************************************************************
4024 static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
4025 //**********************************************************************************************
4026
4027 public:
4028 //**Expression template evaluation functions****************************************************
4031 template< typename Other >
4032 inline bool canAlias( const Other* alias ) const noexcept;
4033
4034 template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
4035 inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
4036
4037 template< typename Other >
4038 inline bool isAliased( const Other* alias ) const noexcept;
4039
4040 template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
4041 inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
4042
4043 inline bool isAligned () const noexcept;
4044 inline bool canSMPAssign() const noexcept;
4045
4046 BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
4047 BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
4048 BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
4049
4050 BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
4051 BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
4052 BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
4053 BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
4054
4055 template< typename MT2 >
4056 inline auto assign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT2> >;
4057
4058 template< typename MT2 >
4059 inline auto assign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT2> >;
4060
4061 template< typename MT2 > inline void assign( const DenseMatrix<MT2,false>& rhs );
4062 template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
4063 template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
4064
4065 template< typename MT2 >
4066 inline auto addAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT2> >;
4067
4068 template< typename MT2 >
4069 inline auto addAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT2> >;
4070
4071 template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,false>& rhs );
4072 template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
4073 template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
4074
4075 template< typename MT2 >
4076 inline auto subAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT2> >;
4077
4078 template< typename MT2 >
4079 inline auto subAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT2> >;
4080
4081 template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,false>& rhs );
4082 template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
4083 template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
4084
4085 template< typename MT2 >
4086 inline auto schurAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT2> >;
4087
4088 template< typename MT2 >
4089 inline auto schurAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT2> >;
4090
4091 template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,false>& rhs );
4092 template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
4093 template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
4095 //**********************************************************************************************
4096
4097 private:
4098 //**Utility functions***************************************************************************
4101 inline bool hasOverlap() const noexcept;
4103 //**********************************************************************************************
4104
4105 //**Member variables****************************************************************************
4108 Operand matrix_;
4109 const bool isAligned_;
4117 //**********************************************************************************************
4118
4119 //**Friend declarations*************************************************************************
4120 template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
4121 //**********************************************************************************************
4122
4123 //**Compile time checks*************************************************************************
4131 //**********************************************************************************************
4132};
4134//*************************************************************************************************
4135
4136
4137
4138
4139//=================================================================================================
4140//
4141// CONSTRUCTORS
4142//
4143//=================================================================================================
4144
4145//*************************************************************************************************
4158template< typename MT // Type of the dense matrix
4159 , size_t... CSAs > // Compile time submatrix arguments
4160template< typename... RSAs > // Runtime submatrix arguments
4161inline Submatrix<MT,unaligned,true,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
4162 : DataType ( args... ) // Base class initialization
4163 , matrix_ ( matrix ) // The matrix containing the submatrix
4164 , isAligned_( simdEnabled && IsContiguous_v<MT> &&
4165 matrix.data() != nullptr && checkAlignment( data() ) &&
4166 ( columns() < 2UL || ( matrix.spacing() % SIMDSIZE ) == 0UL ) )
4167{
4168 if( isChecked( args... ) ) {
4169 if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
4170 BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
4171 }
4172 }
4173 else {
4174 BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
4175 BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
4176 }
4177}
4179//*************************************************************************************************
4180
4181
4182
4183
4184//=================================================================================================
4185//
4186// DATA ACCESS FUNCTIONS
4187//
4188//=================================================================================================
4189
4190//*************************************************************************************************
4201template< typename MT // Type of the dense matrix
4202 , size_t... CSAs > // Compile time submatrix arguments
4203inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Reference
4204 Submatrix<MT,unaligned,true,true,CSAs...>::operator()( size_t i, size_t j )
4205{
4206 BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
4207 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
4208
4209 return matrix_(row()+i,column()+j);
4210}
4212//*************************************************************************************************
4213
4214
4215//*************************************************************************************************
4226template< typename MT // Type of the dense matrix
4227 , size_t... CSAs > // Compile time submatrix arguments
4228inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstReference
4229 Submatrix<MT,unaligned,true,true,CSAs...>::operator()( size_t i, size_t j ) const
4230{
4231 BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
4232 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
4233
4234 return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
4235}
4237//*************************************************************************************************
4238
4239
4240//*************************************************************************************************
4252template< typename MT // Type of the dense matrix
4253 , size_t... CSAs > // Compile time submatrix arguments
4254inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Reference
4255 Submatrix<MT,unaligned,true,true,CSAs...>::at( size_t i, size_t j )
4256{
4257 if( i >= rows() ) {
4258 BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
4259 }
4260 if( j >= columns() ) {
4261 BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
4262 }
4263 return (*this)(i,j);
4264}
4266//*************************************************************************************************
4267
4268
4269//*************************************************************************************************
4281template< typename MT // Type of the dense matrix
4282 , size_t... CSAs > // Compile time submatrix arguments
4283inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstReference
4284 Submatrix<MT,unaligned,true,true,CSAs...>::at( size_t i, size_t j ) const
4285{
4286 if( i >= rows() ) {
4287 BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
4288 }
4289 if( j >= columns() ) {
4290 BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
4291 }
4292 return (*this)(i,j);
4293}
4295//*************************************************************************************************
4296
4297
4298//*************************************************************************************************
4308template< typename MT // Type of the dense matrix
4309 , size_t... CSAs > // Compile time submatrix arguments
4310inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Pointer
4312{
4313 return matrix_.data() + row() + column()*spacing();
4314}
4316//*************************************************************************************************
4317
4318
4319//*************************************************************************************************
4329template< typename MT // Type of the dense matrix
4330 , size_t... CSAs > // Compile time submatrix arguments
4331inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstPointer
4333{
4334 return matrix_.data() + row() + column()*spacing();
4335}
4337//*************************************************************************************************
4338
4339
4340//*************************************************************************************************
4349template< typename MT // Type of the dense matrix
4350 , size_t... CSAs > // Compile time submatrix arguments
4351inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Pointer
4353{
4354 return matrix_.data() + row() + (column()+j)*spacing();
4355}
4357//*************************************************************************************************
4358
4359
4360//*************************************************************************************************
4369template< typename MT // Type of the dense matrix
4370 , size_t... CSAs > // Compile time submatrix arguments
4371inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstPointer
4372 Submatrix<MT,unaligned,true,true,CSAs...>::data( size_t j ) const noexcept
4373{
4374 return matrix_.data() + row() + (column()+j)*spacing();
4375}
4377//*************************************************************************************************
4378
4379
4380//*************************************************************************************************
4387template< typename MT // Type of the dense matrix
4388 , size_t... CSAs > // Compile time submatrix arguments
4389inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Iterator
4391{
4392 BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4393 return Iterator( matrix_.begin( column() + j ) + row(), isAligned_ );
4394}
4396//*************************************************************************************************
4397
4398
4399//*************************************************************************************************
4406template< typename MT // Type of the dense matrix
4407 , size_t... CSAs > // Compile time submatrix arguments
4408inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4410{
4411 BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4412 return ConstIterator( matrix_.cbegin( column() + j ) + row(), isAligned_ );
4413}
4415//*************************************************************************************************
4416
4417
4418//*************************************************************************************************
4425template< typename MT // Type of the dense matrix
4426 , size_t... CSAs > // Compile time submatrix arguments
4427inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4429{
4430 BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4431 return ConstIterator( matrix_.cbegin( column() + j ) + row(), isAligned_ );
4432}
4434//*************************************************************************************************
4435
4436
4437//*************************************************************************************************
4444template< typename MT // Type of the dense matrix
4445 , size_t... CSAs > // Compile time submatrix arguments
4446inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Iterator
4448{
4449 BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4450 return Iterator( matrix_.begin( column() + j ) + row() + rows(), isAligned_ );
4451}
4453//*************************************************************************************************
4454
4455
4456//*************************************************************************************************
4463template< typename MT // Type of the dense matrix
4464 , size_t... CSAs > // Compile time submatrix arguments
4465inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4467{
4468 BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4469 return ConstIterator( matrix_.cbegin( column() + j ) + row() + rows(), isAligned_ );
4470}
4472//*************************************************************************************************
4473
4474
4475//*************************************************************************************************
4482template< typename MT // Type of the dense matrix
4483 , size_t... CSAs > // Compile time submatrix arguments
4484inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4486{
4487 BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4488 return ConstIterator( matrix_.cbegin( column() + j ) + row() + rows(), isAligned_ );
4489}
4491//*************************************************************************************************
4492
4493
4494
4495
4496//=================================================================================================
4497//
4498// ASSIGNMENT OPERATORS
4499//
4500//=================================================================================================
4501
4502//*************************************************************************************************
4513template< typename MT // Type of the dense matrix
4514 , size_t... CSAs > // Compile time submatrix arguments
4515inline Submatrix<MT,unaligned,true,true,CSAs...>&
4516 Submatrix<MT,unaligned,true,true,CSAs...>::operator=( const ElementType& rhs )
4517{
4518 const size_t jend( column() + columns() );
4519 decltype(auto) left( derestrict( matrix_ ) );
4520
4521 for( size_t j=column(); j<jend; ++j )
4522 {
4523 const size_t ibegin( ( IsLower_v<MT> )
4524 ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
4525 ?( max( j+1UL, row() ) )
4526 :( max( j, row() ) ) )
4527 :( row() ) );
4528 const size_t iend ( ( IsUpper_v<MT> )
4529 ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
4530 ?( min( j, row()+rows() ) )
4531 :( min( j+1UL, row()+rows() ) ) )
4532 :( row()+rows() ) );
4533
4534 for( size_t i=ibegin; i<iend; ++i ) {
4535 if( !IsRestricted_v<MT> || IsTriangular_v<MT> || trySet( matrix_, i, j, rhs ) )
4536 left(i,j) = rhs;
4537 }
4538 }
4539
4540 return *this;
4541}
4543//*************************************************************************************************
4544
4545
4546//*************************************************************************************************
4562template< typename MT // Type of the dense matrix
4563 , size_t... CSAs > // Compile time submatrix arguments
4564inline Submatrix<MT,unaligned,true,true,CSAs...>&
4565 Submatrix<MT,unaligned,true,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
4566{
4567 using blaze::reset;
4568
4569 if( list.size() != rows() ) {
4570 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
4571 }
4572
4573 if( IsRestricted_v<MT> ) {
4574 const InitializerMatrix<ElementType> tmp( list, columns() );
4575 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
4576 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4577 }
4578 }
4579
4580 decltype(auto) left( derestrict( *this ) );
4581 size_t i( 0UL );
4582
4583 for( const auto& rowList : list ) {
4584 size_t j( 0UL );
4585 for( const auto& element : rowList ) {
4586 left(i,j) = element;
4587 ++j;
4588 }
4589 for( ; j<columns(); ++j ) {
4590 reset( left(i,j) );
4591 }
4592 ++i;
4593 }
4594
4595 return *this;
4596}
4598//*************************************************************************************************
4599
4600
4601//*************************************************************************************************
4616template< typename MT // Type of the dense matrix
4617 , size_t... CSAs > // Compile time submatrix arguments
4618inline Submatrix<MT,unaligned,true,true,CSAs...>&
4619 Submatrix<MT,unaligned,true,true,CSAs...>::operator=( const Submatrix& rhs )
4620{
4623
4624 if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
4625 return *this;
4626
4627 if( rows() != rhs.rows() || columns() != rhs.columns() ) {
4628 BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
4629 }
4630
4631 if( !tryAssign( matrix_, rhs, row(), column() ) ) {
4632 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4633 }
4634
4635 decltype(auto) left( derestrict( *this ) );
4636
4637 if( rhs.canAlias( this ) ) {
4638 const ResultType tmp( rhs );
4639 smpAssign( left, tmp );
4640 }
4641 else {
4642 smpAssign( left, rhs );
4643 }
4644
4645 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4646
4647 return *this;
4648}
4650//*************************************************************************************************
4651
4652
4653//*************************************************************************************************
4668template< typename MT // Type of the dense matrix
4669 , size_t... CSAs > // Compile time submatrix arguments
4670template< typename MT2 // Type of the right-hand side matrix
4671 , bool SO > // Storage order of the right-hand side matrix
4672inline Submatrix<MT,unaligned,true,true,CSAs...>&
4673 Submatrix<MT,unaligned,true,true,CSAs...>::operator=( const Matrix<MT2,SO>& rhs )
4674{
4676
4677 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
4678 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4679 }
4680
4681 using Right = If_t< IsRestricted_v<MT>, CompositeType_t<MT2>, const MT2& >;
4682 Right right( *rhs );
4683
4684 if( !tryAssign( matrix_, right, row(), column() ) ) {
4685 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4686 }
4687
4688 decltype(auto) left( derestrict( *this ) );
4689
4690 if( IsReference_v<Right> && right.canAlias( this ) ) {
4691 const ResultType_t<MT2> tmp( right );
4692 if( IsSparseMatrix_v<MT2> )
4693 reset();
4694 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
4695 }
4696 else {
4697 if( IsSparseMatrix_v<MT2> )
4698 reset();
4699 smpAssign( left, transIf< IsSymmetric_v<This> >( right ) );
4700 }
4701
4702 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4703
4704 return *this;
4705}
4707//*************************************************************************************************
4708
4709
4710//*************************************************************************************************
4724template< typename MT // Type of the dense matrix
4725 , size_t... CSAs > // Compile time submatrix arguments
4726template< typename MT2 // Type of the right-hand side matrix
4727 , bool SO > // Storage order of the right-hand side matrix
4728inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
4729 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4730{
4734
4735 using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
4736
4739
4740 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
4741 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4742 }
4743
4744 if( !tryAddAssign( matrix_, *rhs, row(), column() ) ) {
4745 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4746 }
4747
4748 decltype(auto) left( derestrict( *this ) );
4749
4750 if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) || (*rhs).canAlias( this ) ) {
4751 const AddType tmp( *this + (*rhs) );
4752 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
4753 }
4754 else {
4755 smpAddAssign( left, transIf< IsSymmetric_v<This> >( *rhs ) );
4756 }
4757
4758 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4759
4760 return *this;
4761}
4763//*************************************************************************************************
4764
4765
4766//*************************************************************************************************
4780template< typename MT // Type of the dense matrix
4781 , size_t... CSAs > // Compile time submatrix arguments
4782template< typename MT2 // Type of the right-hand side matrix
4783 , bool SO > // Storage order of the right-hand side matrix
4784inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
4785 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4786{
4790
4791 using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
4792
4795
4796 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
4797 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4798 }
4799
4800 const AddType tmp( *this + (*rhs) );
4801
4802 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
4803 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4804 }
4805
4806 decltype(auto) left( derestrict( *this ) );
4807
4808 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
4809
4810 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4811
4812 return *this;
4813}
4815//*************************************************************************************************
4816
4817
4818//*************************************************************************************************
4832template< typename MT // Type of the dense matrix
4833 , size_t... CSAs > // Compile time submatrix arguments
4834template< typename MT2 // Type of the right-hand side matrix
4835 , bool SO > // Storage order of the right-hand side matrix
4836inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
4837 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4838{
4842
4843 using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
4844
4847
4848 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
4849 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4850 }
4851
4852 if( !trySubAssign( matrix_, *rhs, row(), column() ) ) {
4853 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4854 }
4855
4856 decltype(auto) left( derestrict( *this ) );
4857
4858 if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) || (*rhs).canAlias( this ) ) {
4859 const SubType tmp( *this - (*rhs ) );
4860 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
4861 }
4862 else {
4863 smpSubAssign( left, transIf< IsSymmetric_v<This> >( *rhs ) );
4864 }
4865
4866 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4867
4868 return *this;
4869}
4871//*************************************************************************************************
4872
4873
4874//*************************************************************************************************
4888template< typename MT // Type of the dense matrix
4889 , size_t... CSAs > // Compile time submatrix arguments
4890template< typename MT2 // Type of the right-hand side matrix
4891 , bool SO > // Storage order of the right-hand side matrix
4892inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
4893 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4894{
4898
4899 using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
4900
4903
4904 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
4905 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4906 }
4907
4908 const SubType tmp( *this - (*rhs) );
4909
4910 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
4911 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4912 }
4913
4914 decltype(auto) left( derestrict( *this ) );
4915
4916 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
4917
4918 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4919
4920 return *this;
4921}
4923//*************************************************************************************************
4924
4925
4926//*************************************************************************************************
4940template< typename MT // Type of the dense matrix
4941 , size_t... CSAs > // Compile time submatrix arguments
4942template< typename MT2 // Type of the right-hand side matrix
4943 , bool SO > // Storage order of the right-hand side matrix
4944inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
4945 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4946{
4950
4951 using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
4952
4954
4955 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
4956 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4957 }
4958
4959 if( !trySchurAssign( matrix_, *rhs, row(), column() ) ) {
4960 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4961 }
4962
4963 decltype(auto) left( derestrict( *this ) );
4964
4965 if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) || (*rhs).canAlias( this ) ) {
4966 const SchurType tmp( *this % (*rhs) );
4967 if( IsSparseMatrix_v<SchurType> )
4968 reset();
4969 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
4970 }
4971 else {
4972 smpSchurAssign( left, transIf< IsSymmetric_v<This> >( *rhs ) );
4973 }
4974
4975 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4976
4977 return *this;
4978}
4980//*************************************************************************************************
4981
4982
4983//*************************************************************************************************
4997template< typename MT // Type of the dense matrix
4998 , size_t... CSAs > // Compile time submatrix arguments
4999template< typename MT2 // Type of the right-hand side matrix
5000 , bool SO > // Storage order of the right-hand side matrix
5001inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
5002 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
5003{
5007
5008 using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
5009
5011
5012 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
5013 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
5014 }
5015
5016 const SchurType tmp( *this % (*rhs) );
5017
5018 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
5019 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
5020 }
5021
5022 decltype(auto) left( derestrict( *this ) );
5023
5024 if( IsSparseMatrix_v<SchurType> ) {
5025 reset();
5026 }
5027
5028 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
5029
5030 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
5031
5032 return *this;
5033}
5035//*************************************************************************************************
5036
5037
5038
5039
5040//=================================================================================================
5041//
5042// UTILITY FUNCTIONS
5043//
5044//=================================================================================================
5045
5046//*************************************************************************************************
5052template< typename MT // Type of the dense matrix
5053 , size_t... CSAs > // Compile time submatrix arguments
5054inline MT& Submatrix<MT,unaligned,true,true,CSAs...>::operand() noexcept
5055{
5056 return matrix_;
5057}
5059//*************************************************************************************************
5060
5061
5062//*************************************************************************************************
5068template< typename MT // Type of the dense matrix
5069 , size_t... CSAs > // Compile time submatrix arguments
5070inline const MT& Submatrix<MT,unaligned,true,true,CSAs...>::operand() const noexcept
5071{
5072 return matrix_;
5073}
5075//*************************************************************************************************
5076
5077
5078//*************************************************************************************************
5087template< typename MT // Type of the dense matrix
5088 , size_t... CSAs > // Compile time submatrix arguments
5089inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::spacing() const noexcept
5090{
5091 return matrix_.spacing();
5092}
5094//*************************************************************************************************
5095
5096
5097//*************************************************************************************************
5103template< typename MT // Type of the dense matrix
5104 , size_t... CSAs > // Compile time submatrix arguments
5105inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::capacity() const noexcept
5106{
5107 return rows() * columns();
5108}
5110//*************************************************************************************************
5111
5112
5113//*************************************************************************************************
5120template< typename MT // Type of the dense matrix
5121 , size_t... CSAs > // Compile time submatrix arguments
5122inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::capacity( size_t j ) const noexcept
5123{
5124 MAYBE_UNUSED( j );
5125
5126 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
5127
5128 return rows();
5129}
5131//*************************************************************************************************
5132
5133
5134//*************************************************************************************************
5140template< typename MT // Type of the dense matrix
5141 , size_t... CSAs > // Compile time submatrix arguments
5143{
5144 const size_t iend( row() + rows() );
5145 const size_t jend( column() + columns() );
5146 size_t nonzeros( 0UL );
5147
5148 for( size_t j=column(); j<jend; ++j )
5149 for( size_t i=row(); i<iend; ++i )
5150 if( !isDefault( matrix_(i,j) ) )
5151 ++nonzeros;
5152
5153 return nonzeros;
5154}
5156//*************************************************************************************************
5157
5158
5159//*************************************************************************************************
5166template< typename MT // Type of the dense matrix
5167 , size_t... CSAs > // Compile time submatrix arguments
5168inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::nonZeros( size_t j ) const
5169{
5170 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
5171
5172 const size_t iend( row() + rows() );
5173 size_t nonzeros( 0UL );
5174
5175 for( size_t i=row(); i<iend; ++i )
5176 if( !isDefault( matrix_(i,column()+j) ) )
5177 ++nonzeros;
5178
5179 return nonzeros;
5180}
5182//*************************************************************************************************
5183
5184
5185//*************************************************************************************************
5191template< typename MT // Type of the dense matrix
5192 , size_t... CSAs > // Compile time submatrix arguments
5194{
5195 using blaze::clear;
5196
5197 for( size_t j=column(); j<column()+columns(); ++j )
5198 {
5199 const size_t ibegin( ( IsLower_v<MT> )
5200 ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
5201 ?( max( j+1UL, row() ) )
5202 :( max( j, row() ) ) )
5203 :( row() ) );
5204 const size_t iend ( ( IsUpper_v<MT> )
5205 ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
5206 ?( min( j, row()+rows() ) )
5207 :( min( j+1UL, row()+rows() ) ) )
5208 :( row()+rows() ) );
5209
5210 for( size_t i=ibegin; i<iend; ++i )
5211 clear( matrix_(i,j) );
5212 }
5213}
5215//*************************************************************************************************
5216
5217
5218//*************************************************************************************************
5225template< typename MT // Type of the dense matrix
5226 , size_t... CSAs > // Compile time submatrix arguments
5228{
5229 using blaze::clear;
5230
5231 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
5232
5233 const size_t ibegin( ( IsLower_v<MT> )
5234 ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
5235 ?( max( j+1UL, row() ) )
5236 :( max( j, row() ) ) )
5237 :( row() ) );
5238 const size_t iend ( ( IsUpper_v<MT> )
5239 ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
5240 ?( min( j, row()+rows() ) )
5241 :( min( j+1UL, row()+rows() ) ) )
5242 :( row()+rows() ) );
5243
5244 for( size_t i=ibegin; i<iend; ++i )
5245 clear( matrix_(i,column()+j) );
5246}
5248//*************************************************************************************************
5249
5250
5251//*************************************************************************************************
5261template< typename MT // Type of the dense matrix
5262 , size_t... CSAs > // Compile time submatrix arguments
5263inline bool Submatrix<MT,unaligned,true,true,CSAs...>::hasOverlap() const noexcept
5264{
5265 BLAZE_INTERNAL_ASSERT( IsSymmetric_v<MT> || IsHermitian_v<MT>, "Invalid matrix detected" );
5266
5267 if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
5268 return false;
5269 else return true;
5270}
5272//*************************************************************************************************
5273
5274
5275
5276
5277//=================================================================================================
5278//
5279// NUMERIC FUNCTIONS
5280//
5281//=================================================================================================
5282
5283//*************************************************************************************************
5301template< typename MT // Type of the dense matrix
5302 , size_t... CSAs > // Compile time submatrix arguments
5303inline Submatrix<MT,unaligned,true,true,CSAs...>&
5305{
5306 if( rows() != columns() ) {
5307 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
5308 }
5309
5310 if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
5311 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
5312 }
5313
5314 decltype(auto) left( derestrict( *this ) );
5315 const ResultType tmp( trans( *this ) );
5316
5317 smpAssign( left, tmp );
5318
5319 return *this;
5320}
5322//*************************************************************************************************
5323
5324
5325//*************************************************************************************************
5343template< typename MT // Type of the dense matrix
5344 , size_t... CSAs > // Compile time submatrix arguments
5345inline Submatrix<MT,unaligned,true,true,CSAs...>&
5347{
5348 if( rows() != columns() ) {
5349 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
5350 }
5351
5352 if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
5353 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
5354 }
5355
5356 decltype(auto) left( derestrict( *this ) );
5357 const ResultType tmp( ctrans( *this ) );
5358
5359 smpAssign( left, tmp );
5360
5361 return *this;
5362}
5364//*************************************************************************************************
5365
5366
5367//*************************************************************************************************
5380template< typename MT // Type of the dense matrix
5381 , size_t... CSAs > // Compile time submatrix arguments
5382template< typename Other > // Data type of the scalar value
5383inline Submatrix<MT,unaligned,true,true,CSAs...>&
5384 Submatrix<MT,unaligned,true,true,CSAs...>::scale( const Other& scalar )
5385{
5387
5388 const size_t jend( column() + columns() );
5389
5390 for( size_t j=column(); j<jend; ++j )
5391 {
5392 const size_t ibegin( ( IsLower_v<MT> )
5393 ?( ( IsStrictlyLower_v<MT> )
5394 ?( max( j+1UL, row() ) )
5395 :( max( j, row() ) ) )
5396 :( row() ) );
5397 const size_t iend ( ( IsUpper_v<MT> )
5398 ?( ( IsStrictlyUpper_v<MT> )
5399 ?( min( j, row()+rows() ) )
5400 :( min( j+1UL, row()+rows() ) ) )
5401 :( row()+rows() ) );
5402
5403 for( size_t i=ibegin; i<iend; ++i )
5404 matrix_(i,j) *= scalar;
5405 }
5406
5407 return *this;
5408}
5410//*************************************************************************************************
5411
5412
5413
5414
5415//=================================================================================================
5416//
5417// EXPRESSION TEMPLATE EVALUATION FUNCTIONS
5418//
5419//=================================================================================================
5420
5421//*************************************************************************************************
5432template< typename MT // Type of the dense matrix
5433 , size_t... CSAs > // Compile time submatrix arguments
5434template< typename Other > // Data type of the foreign expression
5435inline bool Submatrix<MT,unaligned,true,true,CSAs...>::canAlias( const Other* alias ) const noexcept
5436{
5437 return matrix_.isAliased( &unview( *alias ) );
5438}
5440//*************************************************************************************************
5441
5442
5443//*************************************************************************************************
5454template< typename MT // Type of the dense matrix
5455 , size_t... CSAs > // Compile time submatrix arguments
5456template< typename MT2 // Data type of the foreign dense submatrix
5457 , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
5458 , bool SO2 // Storage order of the foreign dense submatrix
5459 , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
5460inline bool
5461 Submatrix<MT,unaligned,true,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
5462{
5463 return ( matrix_.isAliased( &alias->matrix_ ) &&
5464 ( row() + rows() > alias->row() ) &&
5465 ( row() < alias->row() + alias->rows() ) &&
5466 ( column() + columns() > alias->column() ) &&
5467 ( column() < alias->column() + alias->columns() ) );
5468}
5470//*************************************************************************************************
5471
5472
5473//*************************************************************************************************
5484template< typename MT // Type of the dense matrix
5485 , size_t... CSAs > // Compile time submatrix arguments
5486template< typename Other > // Data type of the foreign expression
5487inline bool Submatrix<MT,unaligned,true,true,CSAs...>::isAliased( const Other* alias ) const noexcept
5488{
5489 return matrix_.isAliased( &unview( *alias ) );
5490}
5492//*************************************************************************************************
5493
5494
5495//*************************************************************************************************
5506template< typename MT // Type of the dense matrix
5507 , size_t... CSAs > // Compile time submatrix arguments
5508template< typename MT2 // Data type of the foreign dense submatrix
5509 , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
5510 , bool SO2 // Storage order of the foreign dense submatrix
5511 , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
5512inline bool
5513 Submatrix<MT,unaligned,true,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
5514{
5515 return ( matrix_.isAliased( &alias->matrix_ ) &&
5516 ( row() + rows() > alias->row() ) &&
5517 ( row() < alias->row() + alias->rows() ) &&
5518 ( column() + columns() > alias->column() ) &&
5519 ( column() < alias->column() + alias->columns() ) );
5520}
5522//*************************************************************************************************
5523
5524
5525//*************************************************************************************************
5535template< typename MT // Type of the dense matrix
5536 , size_t... CSAs > // Compile time submatrix arguments
5537inline bool Submatrix<MT,unaligned,true,true,CSAs...>::isAligned() const noexcept
5538{
5539 return isAligned_;
5540}
5542//*************************************************************************************************
5543
5544
5545//*************************************************************************************************
5556template< typename MT // Type of the dense matrix
5557 , size_t... CSAs > // Compile time submatrix arguments
5558inline bool Submatrix<MT,unaligned,true,true,CSAs...>::canSMPAssign() const noexcept
5559{
5560 return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
5561}
5563//*************************************************************************************************
5564
5565
5566//*************************************************************************************************
5581template< typename MT // Type of the dense matrix
5582 , size_t... CSAs > // Compile time submatrix arguments
5583BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,true,true,CSAs...>::SIMDType
5584 Submatrix<MT,unaligned,true,true,CSAs...>::load( size_t i, size_t j ) const noexcept
5585{
5586 if( isAligned_ )
5587 return loada( i, j );
5588 else
5589 return loadu( i, j );
5590}
5592//*************************************************************************************************
5593
5594
5595//*************************************************************************************************
5610template< typename MT // Type of the dense matrix
5611 , size_t... CSAs > // Compile time submatrix arguments
5612BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,true,true,CSAs...>::SIMDType
5613 Submatrix<MT,unaligned,true,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
5614{
5616
5617 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5618 BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5619 BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5620 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5621
5622 return matrix_.loada( row()+i, column()+j );
5623}
5625//*************************************************************************************************
5626
5627
5628//*************************************************************************************************
5643template< typename MT // Type of the dense matrix
5644 , size_t... CSAs > // Compile time submatrix arguments
5645BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,true,true,CSAs...>::SIMDType
5646 Submatrix<MT,unaligned,true,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
5647{
5649
5650 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5651 BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5652 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5653
5654 return matrix_.loadu( row()+i, column()+j );
5655}
5657//*************************************************************************************************
5658
5659
5660//*************************************************************************************************
5676template< typename MT // Type of the dense matrix
5677 , size_t... CSAs > // Compile time submatrix arguments
5679 Submatrix<MT,unaligned,true,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
5680{
5681 if( isAligned_ )
5682 storea( i, j, value );
5683 else
5684 storeu( i, j, value );
5685}
5687//*************************************************************************************************
5688
5689
5690//*************************************************************************************************
5706template< typename MT // Type of the dense matrix
5707 , size_t... CSAs > // Compile time submatrix arguments
5709 Submatrix<MT,unaligned,true,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
5710{
5712
5713 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5714 BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5715 BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5716 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5717
5718 matrix_.storea( row()+i, column()+j, value );
5719}
5721//*************************************************************************************************
5722
5723
5724//*************************************************************************************************
5740template< typename MT // Type of the dense matrix
5741 , size_t... CSAs > // Compile time submatrix arguments
5743 Submatrix<MT,unaligned,true,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
5744{
5746
5747 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5748 BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5749 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5750
5751 matrix_.storeu( row()+i, column()+j, value );
5752}
5754//*************************************************************************************************
5755
5756
5757//*************************************************************************************************
5774template< typename MT // Type of the dense matrix
5775 , size_t... CSAs > // Compile time submatrix arguments
5777 Submatrix<MT,unaligned,true,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
5778{
5780
5781 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5782 BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5783 BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5784 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5785
5786 if( isAligned_ )
5787 matrix_.stream( row()+i, column()+j, value );
5788 else
5789 matrix_.storeu( row()+i, column()+j, value );
5790}
5792//*************************************************************************************************
5793
5794
5795//*************************************************************************************************
5807template< typename MT // Type of the dense matrix
5808 , size_t... CSAs > // Compile time submatrix arguments
5809template< typename MT2 > // Type of the right-hand side dense matrix
5810inline auto Submatrix<MT,unaligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
5811 -> DisableIf_t< VectorizedAssign_v<MT2> >
5812{
5813 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
5814 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
5815
5816 const size_t ipos( prevMultiple( rows(), 2UL ) );
5817 BLAZE_INTERNAL_ASSERT( ipos <= rows(), "Invalid end calculation" );
5818
5819 for( size_t j=0UL; j<columns(); ++j ) {
5820 for( size_t i=0UL; i<ipos; i+=2UL ) {
5821 matrix_(row()+i ,column()+j) = (*rhs)(i ,j);
5822 matrix_(row()+i+1UL,column()+j) = (*rhs)(i+1UL,j);
5823 }
5824 if( ipos < rows() ) {
5825 matrix_(row()+ipos,column()+j) = (*rhs)(ipos,j);
5826 }
5827 }
5828}
5830//*************************************************************************************************
5831
5832
5833//*************************************************************************************************
5845template< typename MT // Type of the dense matrix
5846 , size_t... CSAs > // Compile time submatrix arguments
5847template< typename MT2 > // Type of the right-hand side dense matrix
5848inline auto Submatrix<MT,unaligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
5849 -> EnableIf_t< VectorizedAssign_v<MT2> >
5850{
5852
5853 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
5854 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
5855
5856 const size_t ipos( prevMultiple( rows(), SIMDSIZE ) );
5857 BLAZE_INTERNAL_ASSERT( ipos <= rows(), "Invalid end calculation" );
5858
5859 if( useStreaming && isAligned_ &&
5860 rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
5861 !(*rhs).isAliased( this ) )
5862 {
5863 for( size_t j=0UL; j<columns(); ++j )
5864 {
5865 size_t i( 0UL );
5866 Iterator left( begin(j) );
5867 ConstIterator_t<MT2> right( (*rhs).begin(j) );
5868
5869 for( ; i<ipos; i+=SIMDSIZE ) {
5870 left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5871 }
5872 for( ; i<rows(); ++i ) {
5873 *left = *right; ++left; ++right;
5874 }
5875 }
5876 }
5877 else
5878 {
5879 for( size_t j=0UL; j<columns(); ++j )
5880 {
5881 size_t i( 0UL );
5882 Iterator left( begin(j) );
5883 ConstIterator_t<MT2> right( (*rhs).begin(j) );
5884
5885 for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
5886 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5887 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5888 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5889 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5890 }
5891 for( ; i<ipos; i+=SIMDSIZE ) {
5892 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5893 }
5894 for( ; i<rows(); ++i ) {
5895 *left = *right; ++left; ++right;
5896 }
5897 }
5898 }
5899}
5901//*************************************************************************************************
5902
5903
5904//*************************************************************************************************
5916template< typename MT // Type of the dense matrix
5917 , size_t... CSAs > // Compile time submatrix arguments
5918template< typename MT2 > // Type of the right-hand side dense matrix
5919inline void Submatrix<MT,unaligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
5920{
5922
5923 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
5924 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
5925
5926 constexpr size_t block( BLOCK_SIZE );
5927
5928 for( size_t jj=0UL; jj<columns(); jj+=block ) {
5929 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
5930 for( size_t ii=0UL; ii<rows(); ii+=block ) {
5931 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
5932 for( size_t j=jj; j<jend; ++j ) {
5933 for( size_t i=ii; i<iend; ++i ) {
5934 matrix_(row()+i,column()+j) = (*rhs)(i,j);
5935 }
5936 }
5937 }
5938 }
5939}
5941//*************************************************************************************************
5942
5943
5944//*************************************************************************************************
5956template< typename MT // Type of the dense matrix
5957 , size_t... CSAs > // Compile time submatrix arguments
5958template< typename MT2 > // Type of the right-hand side sparse matrix
5959inline void Submatrix<MT,unaligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
5960{
5961 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
5962 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
5963
5964 for( size_t j=0UL; j<columns(); ++j )
5965 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element )
5966 matrix_(row()+element->index(),column()+j) = element->value();
5967}
5969//*************************************************************************************************
5970
5971
5972//*************************************************************************************************
5984template< typename MT // Type of the dense matrix
5985 , size_t... CSAs > // Compile time submatrix arguments
5986template< typename MT2 > // Type of the right-hand side sparse matrix
5987inline void Submatrix<MT,unaligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
5988{
5990
5991 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
5992 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
5993
5994 for( size_t i=0UL; i<rows(); ++i )
5995 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element )
5996 matrix_(row()+i,column()+element->index()) = element->value();
5997}
5999//*************************************************************************************************
6000
6001
6002//*************************************************************************************************
6014template< typename MT // Type of the dense matrix
6015 , size_t... CSAs > // Compile time submatrix arguments
6016template< typename MT2 > // Type of the right-hand side dense matrix
6017inline auto Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
6018 -> DisableIf_t< VectorizedAddAssign_v<MT2> >
6019{
6020 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6021 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6022
6023 const size_t ipos( prevMultiple( rows(), 2UL ) );
6024 BLAZE_INTERNAL_ASSERT( ipos <= rows(), "Invalid end calculation" );
6025
6026 for( size_t j=0UL; j<columns(); ++j )
6027 {
6028 if( IsDiagonal_v<MT2> ) {
6029 matrix_(row()+j,column()+j) += (*rhs)(j,j);
6030 }
6031 else {
6032 for( size_t i=0UL; i<ipos; i+=2UL ) {
6033 matrix_(row()+i ,column()+j) += (*rhs)(i ,j);
6034 matrix_(row()+i+1UL,column()+j) += (*rhs)(i+1UL,j);
6035 }
6036 if( ipos < rows() ) {
6037 matrix_(row()+ipos,column()+j) += (*rhs)(ipos,j);
6038 }
6039 }
6040 }
6041}
6043//*************************************************************************************************
6044
6045
6046//*************************************************************************************************
6058template< typename MT // Type of the dense matrix
6059 , size_t... CSAs > // Compile time submatrix arguments
6060template< typename MT2 > // Type of the right-hand side dense matrix
6061inline auto Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
6062 -> EnableIf_t< VectorizedAddAssign_v<MT2> >
6063{
6065
6066 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6067 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6068
6069 for( size_t j=0UL; j<columns(); ++j )
6070 {
6071 const size_t ibegin( ( IsLower_v<MT> )
6072 ?( prevMultiple( ( IsStrictlyLower_v<MT> ? j+1UL : j ), SIMDSIZE ) )
6073 :( 0UL ) );
6074 const size_t iend ( ( IsUpper_v<MT> )
6075 ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
6076 :( rows() ) );
6077 BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
6078
6079 const size_t ipos( prevMultiple( iend, SIMDSIZE ) );
6080 BLAZE_INTERNAL_ASSERT( ipos <= iend, "Invalid end calculation" );
6081
6082 size_t i( ibegin );
6083 Iterator left( begin(j) + ibegin );
6084 ConstIterator_t<MT2> right( (*rhs).begin(j) + ibegin );
6085
6086 for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
6087 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6088 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6089 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6090 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6091 }
6092 for( ; i<ipos; i+=SIMDSIZE ) {
6093 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6094 }
6095 for( ; i<iend; ++i ) {
6096 *left += *right; ++left; ++right;
6097 }
6098 }
6099}
6101//*************************************************************************************************
6102
6103
6104//*************************************************************************************************
6116template< typename MT // Type of the dense matrix
6117 , size_t... CSAs > // Compile time submatrix arguments
6118template< typename MT2 > // Type of the right-hand side dense matrix
6119inline void Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
6120{
6122
6123 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6124 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6125
6126 constexpr size_t block( BLOCK_SIZE );
6127
6128 for( size_t jj=0UL; jj<columns(); jj+=block ) {
6129 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
6130 for( size_t ii=0UL; ii<rows(); ii+=block ) {
6131 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
6132 for( size_t j=jj; j<jend; ++j ) {
6133 for( size_t i=ii; i<iend; ++i ) {
6134 matrix_(row()+i,column()+j) += (*rhs)(i,j);
6135 }
6136 }
6137 }
6138 }
6139}
6141//*************************************************************************************************
6142
6143
6144//*************************************************************************************************
6156template< typename MT // Type of the dense matrix
6157 , size_t... CSAs > // Compile time submatrix arguments
6158template< typename MT2 > // Type of the right-hand side sparse matrix
6159inline void Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
6160{
6161 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6162 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6163
6164 for( size_t j=0UL; j<columns(); ++j )
6165 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element )
6166 matrix_(row()+element->index(),column()+j) += element->value();
6167}
6169//*************************************************************************************************
6170
6171
6172//*************************************************************************************************
6184template< typename MT // Type of the dense matrix
6185 , size_t... CSAs > // Compile time submatrix arguments
6186template< typename MT2 > // Type of the right-hand side sparse matrix
6187inline void Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
6188{
6190
6191 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6192 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6193
6194 for( size_t i=0UL; i<rows(); ++i )
6195 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element )
6196 matrix_(row()+i,column()+element->index()) += element->value();
6197}
6199//*************************************************************************************************
6200
6201
6202//*************************************************************************************************
6214template< typename MT // Type of the dense matrix
6215 , size_t... CSAs > // Compile time submatrix arguments
6216template< typename MT2 > // Type of the right-hand side dense matrix
6217inline auto Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
6218 -> DisableIf_t< VectorizedSubAssign_v<MT2> >
6219{
6220 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6221 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6222
6223 const size_t ipos( prevMultiple( rows(), 2UL ) );
6224 BLAZE_INTERNAL_ASSERT( ipos <= rows(), "Invalid end calculation" );
6225
6226 for( size_t j=0UL; j<columns(); ++j )
6227 {
6228 if( IsDiagonal_v<MT2> ) {
6229 matrix_(row()+j,column()+j) -= (*rhs)(j,j);
6230 }
6231 else {
6232 for( size_t i=0UL; i<ipos; i+=2UL ) {
6233 matrix_(row()+i ,column()+j) -= (*rhs)(i ,j);
6234 matrix_(row()+i+1UL,column()+j) -= (*rhs)(i+1UL,j);
6235 }
6236 if( ipos < rows() ) {
6237 matrix_(row()+ipos,column()+j) -= (*rhs)(ipos,j);
6238 }
6239 }
6240 }
6241}
6243//*************************************************************************************************
6244
6245
6246//*************************************************************************************************
6258template< typename MT // Type of the dense matrix
6259 , size_t... CSAs > // Compile time submatrix arguments
6260template< typename MT2 > // Type of the right-hand side dense matrix
6261inline auto Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
6262 -> EnableIf_t< VectorizedSubAssign_v<MT2> >
6263{
6265
6266 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6267 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6268
6269 for( size_t j=0UL; j<columns(); ++j )
6270 {
6271 const size_t ibegin( ( IsLower_v<MT> )
6272 ?( prevMultiple( ( IsStrictlyLower_v<MT> ? j+1UL : j ), SIMDSIZE ) )
6273 :( 0UL ) );
6274 const size_t iend ( ( IsUpper_v<MT> )
6275 ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
6276 :( rows() ) );
6277 BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
6278
6279 const size_t ipos( prevMultiple( iend, SIMDSIZE ) );
6280 BLAZE_INTERNAL_ASSERT( ipos <= iend, "Invalid end calculation" );
6281
6282 size_t i( ibegin );
6283 Iterator left( begin(j) + ibegin );
6284 ConstIterator_t<MT2> right( (*rhs).begin(j) + ibegin );
6285
6286 for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
6287 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6288 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6289 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6290 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6291 }
6292 for( ; i<ipos; i+=SIMDSIZE ) {
6293 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6294 }
6295 for( ; i<iend; ++i ) {
6296 *left -= *right; ++left; ++right;
6297 }
6298 }
6299}
6301//*************************************************************************************************
6302
6303
6304//*************************************************************************************************
6316template< typename MT // Type of the dense matrix
6317 , size_t... CSAs > // Compile time submatrix arguments
6318template< typename MT2 > // Type of the right-hand side dense matrix
6319inline void Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
6320{
6322
6323 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6324 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6325
6326 constexpr size_t block( BLOCK_SIZE );
6327
6328 for( size_t jj=0UL; jj<columns(); jj+=block ) {
6329 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
6330 for( size_t ii=0UL; ii<rows(); ii+=block ) {
6331 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
6332 for( size_t j=jj; j<jend; ++j ) {
6333 for( size_t i=ii; i<iend; ++i ) {
6334 matrix_(row()+i,column()+j) -= (*rhs)(i,j);
6335 }
6336 }
6337 }
6338 }
6339}
6341//*************************************************************************************************
6342
6343
6344//*************************************************************************************************
6356template< typename MT // Type of the dense matrix
6357 , size_t... CSAs > // Compile time submatrix arguments
6358template< typename MT2 > // Type of the right-hand side sparse matrix
6359inline void Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
6360{
6361 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6362 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6363
6364 for( size_t j=0UL; j<columns(); ++j )
6365 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element )
6366 matrix_(row()+element->index(),column()+j) -= element->value();
6367}
6369//*************************************************************************************************
6370
6371
6372//*************************************************************************************************
6384template< typename MT // Type of the dense matrix
6385 , size_t... CSAs > // Compile time submatrix arguments
6386template< typename MT2 > // Type of the right-hand side sparse matrix
6387inline void Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
6388{
6390
6391 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6392 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6393
6394 for( size_t i=0UL; i<rows(); ++i )
6395 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element )
6396 matrix_(row()+i,column()+element->index()) -= element->value();
6397}
6399//*************************************************************************************************
6400
6401
6402//*************************************************************************************************
6414template< typename MT // Type of the dense matrix
6415 , size_t... CSAs > // Compile time submatrix arguments
6416template< typename MT2 > // Type of the right-hand side dense matrix
6417inline auto Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
6418 -> DisableIf_t< VectorizedSchurAssign_v<MT2> >
6419{
6420 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6421 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6422
6423 const size_t ipos( prevMultiple( rows(), 2UL ) );
6424 BLAZE_INTERNAL_ASSERT( ipos <= rows(), "Invalid end calculation" );
6425
6426 for( size_t j=0UL; j<columns(); ++j ) {
6427 for( size_t i=0UL; i<ipos; i+=2UL ) {
6428 matrix_(row()+i ,column()+j) *= (*rhs)(i ,j);
6429 matrix_(row()+i+1UL,column()+j) *= (*rhs)(i+1UL,j);
6430 }
6431 if( ipos < rows() ) {
6432 matrix_(row()+ipos,column()+j) *= (*rhs)(ipos,j);
6433 }
6434 }
6435}
6437//*************************************************************************************************
6438
6439
6440//*************************************************************************************************
6453template< typename MT // Type of the dense matrix
6454 , size_t... CSAs > // Compile time submatrix arguments
6455template< typename MT2 > // Type of the right-hand side dense matrix
6456inline auto Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
6457 -> EnableIf_t< VectorizedSchurAssign_v<MT2> >
6458{
6460
6461 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6462 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6463
6464 for( size_t j=0UL; j<columns(); ++j )
6465 {
6466 const size_t ipos( prevMultiple( rows(), SIMDSIZE ) );
6467 BLAZE_INTERNAL_ASSERT( ipos <= rows(), "Invalid end calculation" );
6468
6469 size_t i( 0UL );
6470 Iterator left( begin(j) );
6471 ConstIterator_t<MT2> right( (*rhs).begin(j) );
6472
6473 for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
6474 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6475 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6476 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6477 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6478 }
6479 for( ; i<ipos; i+=SIMDSIZE ) {
6480 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6481 }
6482 for( ; i<rows(); ++i ) {
6483 *left *= *right; ++left; ++right;
6484 }
6485 }
6486}
6488//*************************************************************************************************
6489
6490
6491//*************************************************************************************************
6503template< typename MT // Type of the dense matrix
6504 , size_t... CSAs > // Compile time submatrix arguments
6505template< typename MT2 > // Type of the right-hand side dense matrix
6506inline void Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
6507{
6509
6510 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6511 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6512
6513 constexpr size_t block( BLOCK_SIZE );
6514
6515 for( size_t jj=0UL; jj<columns(); jj+=block ) {
6516 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
6517 for( size_t ii=0UL; ii<rows(); ii+=block ) {
6518 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
6519 for( size_t j=jj; j<jend; ++j ) {
6520 for( size_t i=ii; i<iend; ++i ) {
6521 matrix_(row()+i,column()+j) *= (*rhs)(i,j);
6522 }
6523 }
6524 }
6525 }
6526}
6528//*************************************************************************************************
6529
6530
6531//*************************************************************************************************
6543template< typename MT // Type of the dense matrix
6544 , size_t... CSAs > // Compile time submatrix arguments
6545template< typename MT2 > // Type of the right-hand side sparse matrix
6546inline void Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
6547{
6548 using blaze::reset;
6549
6550 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6551 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6552
6553 for( size_t j=0UL; j<columns(); ++j )
6554 {
6555 size_t i( 0UL );
6556
6557 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element ) {
6558 for( ; i<element->index(); ++i )
6559 reset( matrix_(row()+i,column()+j) );
6560 matrix_(row()+i,column()+j) *= element->value();
6561 ++i;
6562 }
6563
6564 for( ; i<rows(); ++i ) {
6565 reset( matrix_(row()+i,column()+j) );
6566 }
6567 }
6568}
6570//*************************************************************************************************
6571
6572
6573//*************************************************************************************************
6585template< typename MT // Type of the dense matrix
6586 , size_t... CSAs > // Compile time submatrix arguments
6587template< typename MT2 > // Type of the right-hand side sparse matrix
6588inline void Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
6589{
6590 using blaze::reset;
6591
6593
6594 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
6595 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
6596
6597 for( size_t i=0UL; i<rows(); ++i )
6598 {
6599 size_t j( 0UL );
6600
6601 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element ) {
6602 for( ; j<element->index(); ++j )
6603 reset( matrix_(row()+i,column()+j) );
6604 matrix_(row()+i,column()+j) *= element->value();
6605 ++j;
6606 }
6607
6608 for( ; j<columns(); ++j ) {
6609 reset( matrix_(row()+i,column()+j) );
6610 }
6611 }
6612}
6614//*************************************************************************************************
6615
6616
6617
6618
6619
6620
6621
6622
6623//=================================================================================================
6624//
6625// CLASS TEMPLATE SPECIALIZATION FOR ALIGNED ROW-MAJOR DENSE SUBMATRICES
6626//
6627//=================================================================================================
6628
6629//*************************************************************************************************
6637template< typename MT // Type of the dense matrix
6638 , size_t... CSAs > // Compile time submatrix arguments
6639class Submatrix<MT,aligned,false,true,CSAs...>
6640 : public View< DenseMatrix< Submatrix<MT,aligned,false,true,CSAs...>, false > >
6641 , private SubmatrixData<CSAs...>
6642{
6643 private:
6644 //**Type definitions****************************************************************************
6645 using DataType = SubmatrixData<CSAs...>;
6646 using Operand = If_t< IsExpression_v<MT>, MT, MT& >;
6647 //**********************************************************************************************
6648
6649 //**********************************************************************************************
6651 template< typename MT1, typename MT2 >
6652 static constexpr bool EnforceEvaluation_v =
6653 ( IsRestricted_v<MT1> && RequiresEvaluation_v<MT2> );
6654 //**********************************************************************************************
6655
6656 public:
6657 //**Type definitions****************************************************************************
6659 using This = Submatrix<MT,aligned,false,true,CSAs...>;
6660
6662 using BaseType = View< DenseMatrix<This,false> >;
6663
6664 using ViewedType = MT;
6665 using ResultType = SubmatrixTrait_t<MT,CSAs...>;
6666 using OppositeType = OppositeType_t<ResultType>;
6667 using TransposeType = TransposeType_t<ResultType>;
6668 using ElementType = ElementType_t<MT>;
6669 using SIMDType = SIMDTrait_t<ElementType>;
6670 using ReturnType = ReturnType_t<MT>;
6671 using CompositeType = const Submatrix&;
6672
6674 using ConstReference = ConstReference_t<MT>;
6675
6677 using Reference = If_t< IsConst_v<MT>, ConstReference, Reference_t<MT> >;
6678
6680 using ConstPointer = ConstPointer_t<MT>;
6681
6683 using Pointer = If_t< IsConst_v<MT> || !HasMutableDataAccess_v<MT>, ConstPointer, Pointer_t<MT> >;
6684
6686 using ConstIterator = ConstIterator_t<MT>;
6687
6689 using Iterator = If_t< IsConst_v<MT>, ConstIterator, Iterator_t<MT> >;
6690 //**********************************************************************************************
6691
6692 //**Compilation flags***************************************************************************
6694 static constexpr bool simdEnabled = MT::simdEnabled;
6695
6697 static constexpr bool smpAssignable = MT::smpAssignable;
6698
6700 static constexpr bool compileTimeArgs = DataType::compileTimeArgs;
6701 //**********************************************************************************************
6702
6703 //**Constructors********************************************************************************
6706 template< typename... RSAs >
6707 explicit inline Submatrix( MT& matrix, RSAs... args );
6708
6709 Submatrix( const Submatrix& ) = default;
6711 //**********************************************************************************************
6712
6713 //**Destructor**********************************************************************************
6716 ~Submatrix() = default;
6718 //**********************************************************************************************
6719
6720 //**Data access functions***********************************************************************
6723 inline Reference operator()( size_t i, size_t j );
6724 inline ConstReference operator()( size_t i, size_t j ) const;
6725 inline Reference at( size_t i, size_t j );
6726 inline ConstReference at( size_t i, size_t j ) const;
6727 inline Pointer data () noexcept;
6728 inline ConstPointer data () const noexcept;
6729 inline Pointer data ( size_t i ) noexcept;
6730 inline ConstPointer data ( size_t i ) const noexcept;
6731 inline Iterator begin ( size_t i );
6732 inline ConstIterator begin ( size_t i ) const;
6733 inline ConstIterator cbegin( size_t i ) const;
6734 inline Iterator end ( size_t i );
6735 inline ConstIterator end ( size_t i ) const;
6736 inline ConstIterator cend ( size_t i ) const;
6738 //**********************************************************************************************
6739
6740 //**Assignment operators************************************************************************
6743 inline Submatrix& operator=( const ElementType& rhs );
6744 inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
6745 inline Submatrix& operator=( const Submatrix& rhs );
6746
6747 template< typename MT2, bool SO >
6748 inline Submatrix& operator=( const Matrix<MT2,SO>& rhs );
6749
6750 template< typename MT2, bool SO >
6751 inline auto operator+=( const Matrix<MT2,SO>& rhs )
6752 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6753
6754 template< typename MT2, bool SO >
6755 inline auto operator+=( const Matrix<MT2,SO>& rhs )
6756 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6757
6758 template< typename MT2, bool SO >
6759 inline auto operator-=( const Matrix<MT2,SO>& rhs )
6760 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6761
6762 template< typename MT2, bool SO >
6763 inline auto operator-=( const Matrix<MT2,SO>& rhs )
6764 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6765
6766 template< typename MT2, bool SO >
6767 inline auto operator%=( const Matrix<MT2,SO>& rhs )
6768 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6769
6770 template< typename MT2, bool SO >
6771 inline auto operator%=( const Matrix<MT2,SO>& rhs )
6772 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6774 //**********************************************************************************************
6775
6776 //**Utility functions***************************************************************************
6779 using DataType::row;
6780 using DataType::column;
6781 using DataType::rows;
6782 using DataType::columns;
6783
6784 inline MT& operand() noexcept;
6785 inline const MT& operand() const noexcept;
6786
6787 inline size_t spacing() const noexcept;
6788 inline size_t capacity() const noexcept;
6789 inline size_t capacity( size_t i ) const noexcept;
6790 inline size_t nonZeros() const;
6791 inline size_t nonZeros( size_t i ) const;
6792 inline void reset();
6793 inline void reset( size_t i );
6795 //**********************************************************************************************
6796
6797 //**Numeric functions***************************************************************************
6800 inline Submatrix& transpose();
6801 inline Submatrix& ctranspose();
6802
6803 template< typename Other > inline Submatrix& scale( const Other& scalar );
6805 //**********************************************************************************************
6806
6807 private:
6808 //**********************************************************************************************
6810 template< typename MT2 >
6811 static constexpr bool VectorizedAssign_v =
6812 ( useOptimizedKernels &&
6813 simdEnabled && MT2::simdEnabled &&
6814 IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > );
6815 //**********************************************************************************************
6816
6817 //**********************************************************************************************
6819 template< typename MT2 >
6820 static constexpr bool VectorizedAddAssign_v =
6821 ( VectorizedAssign_v<MT2> &&
6822 HasSIMDAdd_v< ElementType, ElementType_t<MT2> > &&
6823 !IsDiagonal_v<MT2> );
6824 //**********************************************************************************************
6825
6826 //**********************************************************************************************
6828 template< typename MT2 >
6829 static constexpr bool VectorizedSubAssign_v =
6830 ( VectorizedAssign_v<MT2> &&
6831 HasSIMDSub_v< ElementType, ElementType_t<MT2> > &&
6832 !IsDiagonal_v<MT2> );
6833 //**********************************************************************************************
6834
6835 //**********************************************************************************************
6837 template< typename MT2 >
6838 static constexpr bool VectorizedSchurAssign_v =
6839 ( VectorizedAssign_v<MT2> &&
6840 HasSIMDMult_v< ElementType, ElementType_t<MT2> > );
6841 //**********************************************************************************************
6842
6843 //**SIMD properties*****************************************************************************
6845 static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
6846 //**********************************************************************************************
6847
6848 public:
6849 //**Expression template evaluation functions****************************************************
6852 template< typename Other >
6853 inline bool canAlias( const Other* alias ) const noexcept;
6854
6855 template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
6856 inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
6857
6858 template< typename Other >
6859 inline bool isAliased( const Other* alias ) const noexcept;
6860
6861 template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
6862 inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
6863
6864 inline bool isAligned () const noexcept;
6865 inline bool canSMPAssign() const noexcept;
6866
6867 BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
6868 BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
6869 BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
6870
6871 BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
6872 BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
6873 BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
6874 BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
6875
6876 template< typename MT2 >
6877 inline auto assign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT2> >;
6878
6879 template< typename MT2 >
6880 inline auto assign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT2> >;
6881
6882 template< typename MT2 > inline void assign( const DenseMatrix<MT2,true>& rhs );
6883 template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
6884 template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
6885
6886 template< typename MT2 >
6887 inline auto addAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT2> >;
6888
6889 template< typename MT2 >
6890 inline auto addAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT2> >;
6891
6892 template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,true>& rhs );
6893 template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
6894 template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
6895
6896 template< typename MT2 >
6897 inline auto subAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT2> >;
6898
6899 template< typename MT2 >
6900 inline auto subAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT2> >;
6901
6902 template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,true>& rhs );
6903 template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
6904 template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
6905
6906 template< typename MT2 >
6907 inline auto schurAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT2> >;
6908
6909 template< typename MT2 >
6910 inline auto schurAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT2> >;
6911
6912 template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,true>& rhs );
6913 template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
6914 template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
6916 //**********************************************************************************************
6917
6918 private:
6919 //**Utility functions***************************************************************************
6922 inline bool hasOverlap() const noexcept;
6924 //**********************************************************************************************
6925
6926 //**Member variables****************************************************************************
6929 Operand matrix_;
6931 //**********************************************************************************************
6932
6933 //**Friend declarations*************************************************************************
6934 template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
6935 //**********************************************************************************************
6936
6937 //**Compile time checks*************************************************************************
6945 //**********************************************************************************************
6946};
6948//*************************************************************************************************
6949
6950
6951
6952
6953//=================================================================================================
6954//
6955// CONSTRUCTORS
6956//
6957//=================================================================================================
6958
6959//*************************************************************************************************
6972template< typename MT // Type of the dense matrix
6973 , size_t... CSAs > // Compile time submatrix arguments
6974template< typename... RSAs > // Runtime submatrix arguments
6975inline Submatrix<MT,aligned,false,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
6976 : DataType( args... ) // Base class initialization
6977 , matrix_ ( matrix ) // The matrix containing the submatrix
6978{
6979 if( isChecked( args... ) )
6980 {
6981 if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
6982 BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
6983 }
6984
6985 if( simdEnabled && IsContiguous_v<MT> &&
6986 ( !checkAlignment( data() ) ||
6987 ( rows() > 1UL && matrix_.spacing() % SIMDSIZE != 0UL ) ) ) {
6988 BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix alignment" );
6989 }
6990 }
6991 else
6992 {
6993 BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
6994 BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
6995
6996 BLAZE_USER_ASSERT( !simdEnabled || !IsContiguous_v<MT> || checkAlignment( data() ), "Invalid submatrix alignment" );
6997 BLAZE_USER_ASSERT( !simdEnabled || !IsContiguous_v<MT> || rows() <= 1UL || matrix_.spacing() % SIMDSIZE == 0UL, "Invalid submatrix alignment" );
6998 }
6999}
7001//*************************************************************************************************
7002
7003
7004
7005
7006//=================================================================================================
7007//
7008// DATA ACCESS FUNCTIONS
7009//
7010//=================================================================================================
7011
7012//*************************************************************************************************
7023template< typename MT // Type of the dense matrix
7024 , size_t... CSAs > // Compile time submatrix arguments
7025inline typename Submatrix<MT,aligned,false,true,CSAs...>::Reference
7026 Submatrix<MT,aligned,false,true,CSAs...>::operator()( size_t i, size_t j )
7027{
7028 BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
7029 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
7030
7031 return matrix_(row()+i,column()+j);
7032}
7034//*************************************************************************************************
7035
7036
7037//*************************************************************************************************
7048template< typename MT // Type of the dense matrix
7049 , size_t... CSAs > // Compile time submatrix arguments
7050inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstReference
7051 Submatrix<MT,aligned,false,true,CSAs...>::operator()( size_t i, size_t j ) const
7052{
7053 BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
7054 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
7055
7056 return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
7057}
7059//*************************************************************************************************
7060
7061
7062//*************************************************************************************************
7074template< typename MT // Type of the dense matrix
7075 , size_t... CSAs > // Compile time submatrix arguments
7076inline typename Submatrix<MT,aligned,false,true,CSAs...>::Reference
7077 Submatrix<MT,aligned,false,true,CSAs...>::at( size_t i, size_t j )
7078{
7079 if( i >= rows() ) {
7080 BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
7081 }
7082 if( j >= columns() ) {
7083 BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
7084 }
7085 return (*this)(i,j);
7086}
7088//*************************************************************************************************
7089
7090
7091//*************************************************************************************************
7103template< typename MT // Type of the dense matrix
7104 , size_t... CSAs > // Compile time submatrix arguments
7105inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstReference
7106 Submatrix<MT,aligned,false,true,CSAs...>::at( size_t i, size_t j ) const
7107{
7108 if( i >= rows() ) {
7109 BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
7110 }
7111 if( j >= columns() ) {
7112 BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
7113 }
7114 return (*this)(i,j);
7115}
7117//*************************************************************************************************
7118
7119
7120//*************************************************************************************************
7130template< typename MT // Type of the dense matrix
7131 , size_t... CSAs > // Compile time submatrix arguments
7132inline typename Submatrix<MT,aligned,false,true,CSAs...>::Pointer
7134{
7135 return matrix_.data() + row()*spacing() + column();
7136}
7138//*************************************************************************************************
7139
7140
7141//*************************************************************************************************
7151template< typename MT // Type of the dense matrix
7152 , size_t... CSAs > // Compile time submatrix arguments
7153inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstPointer
7155{
7156 return matrix_.data() + row()*spacing() + column();
7157}
7159//*************************************************************************************************
7160
7161
7162//*************************************************************************************************
7171template< typename MT // Type of the dense matrix
7172 , size_t... CSAs > // Compile time submatrix arguments
7173inline typename Submatrix<MT,aligned,false,true,CSAs...>::Pointer
7175{
7176 return matrix_.data() + (row()+i)*spacing() + column();
7177}
7179//*************************************************************************************************
7180
7181
7182//*************************************************************************************************
7191template< typename MT // Type of the dense matrix
7192 , size_t... CSAs > // Compile time submatrix arguments
7193inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstPointer
7194 Submatrix<MT,aligned,false,true,CSAs...>::data( size_t i ) const noexcept
7195{
7196 return matrix_.data() + (row()+i)*spacing() + column();
7197}
7199//*************************************************************************************************
7200
7201
7202//*************************************************************************************************
7214template< typename MT // Type of the dense matrix
7215 , size_t... CSAs > // Compile time submatrix arguments
7216inline typename Submatrix<MT,aligned,false,true,CSAs...>::Iterator
7218{
7219 BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7220 return ( matrix_.begin( row() + i ) + column() );
7221}
7223//*************************************************************************************************
7224
7225
7226//*************************************************************************************************
7238template< typename MT // Type of the dense matrix
7239 , size_t... CSAs > // Compile time submatrix arguments
7240inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7242{
7243 BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7244 return ( matrix_.cbegin( row() + i ) + column() );
7245}
7247//*************************************************************************************************
7248
7249
7250//*************************************************************************************************
7262template< typename MT // Type of the dense matrix
7263 , size_t... CSAs > // Compile time submatrix arguments
7264inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7266{
7267 BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7268 return ( matrix_.cbegin( row() + i ) + column() );
7269}
7271//*************************************************************************************************
7272
7273
7274//*************************************************************************************************
7286template< typename MT // Type of the dense matrix
7287 , size_t... CSAs > // Compile time submatrix arguments
7288inline typename Submatrix<MT,aligned,false,true,CSAs...>::Iterator
7290{
7291 BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7292 return ( matrix_.begin( row() + i ) + column() + columns() );
7293}
7295//*************************************************************************************************
7296
7297
7298//*************************************************************************************************
7310template< typename MT // Type of the dense matrix
7311 , size_t... CSAs > // Compile time submatrix arguments
7312inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7314{
7315 BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7316 return ( matrix_.cbegin( row() + i ) + column() + columns() );
7317}
7319//*************************************************************************************************
7320
7321
7322//*************************************************************************************************
7334template< typename MT // Type of the dense matrix
7335 , size_t... CSAs > // Compile time submatrix arguments
7336inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7338{
7339 BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7340 return ( matrix_.cbegin( row() + i ) + column() + columns() );
7341}
7343//*************************************************************************************************
7344
7345
7346
7347
7348//=================================================================================================
7349//
7350// ASSIGNMENT OPERATORS
7351//
7352//=================================================================================================
7353
7354//*************************************************************************************************
7365template< typename MT // Type of the dense matrix
7366 , size_t... CSAs > // Compile time submatrix arguments
7367inline Submatrix<MT,aligned,false,true,CSAs...>&
7368 Submatrix<MT,aligned,false,true,CSAs...>::operator=( const ElementType& rhs )
7369{
7370 const size_t iend( row() + rows() );
7371 decltype(auto) left( derestrict( matrix_ ) );
7372
7373 for( size_t i=row(); i<iend; ++i )
7374 {
7375 const size_t jbegin( ( IsUpper_v<MT> )
7376 ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
7377 ?( max( i+1UL, column() ) )
7378 :( max( i, column() ) ) )
7379 :( column() ) );
7380 const size_t jend ( ( IsLower_v<MT> )
7381 ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
7382 ?( min( i, column()+columns() ) )
7383 :( min( i+1UL, column()+columns() ) ) )
7384 :( column()+columns() ) );
7385
7386 for( size_t j=jbegin; j<jend; ++j ) {
7387 if( !IsRestricted_v<MT> || IsTriangular_v<MT> || trySet( matrix_, i, j, rhs ) )
7388 left(i,j) = rhs;
7389 }
7390 }
7391
7392 return *this;
7393}
7395//*************************************************************************************************
7396
7397
7398//*************************************************************************************************
7414template< typename MT // Type of the dense matrix
7415 , size_t... CSAs > // Compile time submatrix arguments
7416inline Submatrix<MT,aligned,false,true,CSAs...>&
7417 Submatrix<MT,aligned,false,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
7418{
7419 if( list.size() != rows() ) {
7420 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
7421 }
7422
7423 if( IsRestricted_v<MT> ) {
7424 const InitializerMatrix<ElementType> tmp( list, columns() );
7425 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7426 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7427 }
7428 }
7429
7430 decltype(auto) left( derestrict( *this ) );
7431 size_t i( 0UL );
7432
7433 for( const auto& rowList : list ) {
7434 std::fill( std::copy( rowList.begin(), rowList.end(), left.begin(i) ), left.end(i), ElementType() );
7435 ++i;
7436 }
7437
7438 return *this;
7439}
7441//*************************************************************************************************
7442
7443
7444//*************************************************************************************************
7459template< typename MT // Type of the dense matrix
7460 , size_t... CSAs > // Compile time submatrix arguments
7461inline Submatrix<MT,aligned,false,true,CSAs...>&
7462 Submatrix<MT,aligned,false,true,CSAs...>::operator=( const Submatrix& rhs )
7463{
7466
7467 if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
7468 return *this;
7469
7470 if( rows() != rhs.rows() || columns() != rhs.columns() ) {
7471 BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
7472 }
7473
7474 if( !tryAssign( matrix_, rhs, row(), column() ) ) {
7475 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7476 }
7477
7478 decltype(auto) left( derestrict( *this ) );
7479
7480 if( rhs.canAlias( this ) ) {
7481 const ResultType tmp( rhs );
7482 smpAssign( left, tmp );
7483 }
7484 else {
7485 smpAssign( left, rhs );
7486 }
7487
7488 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7489
7490 return *this;
7491}
7493//*************************************************************************************************
7494
7495
7496//*************************************************************************************************
7511template< typename MT // Type of the dense matrix
7512 , size_t... CSAs > // Compile time submatrix arguments
7513template< typename MT2 // Type of the right-hand side matrix
7514 , bool SO > // Storage order of the right-hand side matrix
7515inline Submatrix<MT,aligned,false,true,CSAs...>&
7516 Submatrix<MT,aligned,false,true,CSAs...>::operator=( const Matrix<MT2,SO>& rhs )
7517{
7519
7520 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
7521 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7522 }
7523
7524 using Right = If_t< IsRestricted_v<MT>, CompositeType_t<MT2>, const MT2& >;
7525 Right right( *rhs );
7526
7527 if( !tryAssign( matrix_, right, row(), column() ) ) {
7528 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7529 }
7530
7531 decltype(auto) left( derestrict( *this ) );
7532
7533 if( IsReference_v<Right> && right.canAlias( this ) ) {
7534 const ResultType_t<MT2> tmp( right );
7535 if( IsSparseMatrix_v<MT2> )
7536 reset();
7537 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
7538 }
7539 else {
7540 if( IsSparseMatrix_v<MT2> )
7541 reset();
7542 smpAssign( left, transIf< IsSymmetric_v<This> >( right ) );
7543 }
7544
7545 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7546
7547 return *this;
7548}
7550//*************************************************************************************************
7551
7552
7553//*************************************************************************************************
7567template< typename MT // Type of the dense matrix
7568 , size_t... CSAs > // Compile time submatrix arguments
7569template< typename MT2 // Type of the right-hand side matrix
7570 , bool SO > // Storage order of the right-hand side matrix
7571inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
7572 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7573{
7577
7578 using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
7579
7582
7583 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
7584 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7585 }
7586
7587 if( !tryAddAssign( matrix_, *rhs, row(), column() ) ) {
7588 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7589 }
7590
7591 decltype(auto) left( derestrict( *this ) );
7592
7593 if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) || (*rhs).canAlias( this ) ) {
7594 const AddType tmp( *this + (*rhs) );
7595 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
7596 }
7597 else {
7598 smpAddAssign( left, transIf< IsSymmetric_v<This> >( *rhs ) );
7599 }
7600
7601 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7602
7603 return *this;
7604}
7606//*************************************************************************************************
7607
7608
7609//*************************************************************************************************
7623template< typename MT // Type of the dense matrix
7624 , size_t... CSAs > // Compile time submatrix arguments
7625template< typename MT2 // Type of the right-hand side matrix
7626 , bool SO > // Storage order of the right-hand side matrix
7627inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
7628 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7629{
7633
7634 using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
7635
7638
7639 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
7640 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7641 }
7642
7643 const AddType tmp( *this + (*rhs) );
7644
7645 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7646 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7647 }
7648
7649 decltype(auto) left( derestrict( *this ) );
7650
7651 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
7652
7653 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7654
7655 return *this;
7656}
7658//*************************************************************************************************
7659
7660
7661//*************************************************************************************************
7675template< typename MT // Type of the dense matrix
7676 , size_t... CSAs > // Compile time submatrix arguments
7677template< typename MT2 // Type of the right-hand side matrix
7678 , bool SO > // Storage order of the right-hand side matrix
7679inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
7680 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7681{
7685
7686 using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
7687
7690
7691 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
7692 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7693 }
7694
7695 if( !trySubAssign( matrix_, *rhs, row(), column() ) ) {
7696 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7697 }
7698
7699 decltype(auto) left( derestrict( *this ) );
7700
7701 if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) || (*rhs).canAlias( this ) ) {
7702 const SubType tmp( *this - (*rhs ) );
7703 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
7704 }
7705 else {
7706 smpSubAssign( left, transIf< IsSymmetric_v<This> >( *rhs ) );
7707 }
7708
7709 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7710
7711 return *this;
7712}
7714//*************************************************************************************************
7715
7716
7717//*************************************************************************************************
7731template< typename MT // Type of the dense matrix
7732 , size_t... CSAs > // Compile time submatrix arguments
7733template< typename MT2 // Type of the right-hand side matrix
7734 , bool SO > // Storage order of the right-hand side matrix
7735inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
7736 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7737{
7741
7742 using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
7743
7746
7747 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
7748 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7749 }
7750
7751 const SubType tmp( *this - (*rhs) );
7752
7753 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7754 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7755 }
7756
7757 decltype(auto) left( derestrict( *this ) );
7758
7759 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
7760
7761 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7762
7763 return *this;
7764}
7766//*************************************************************************************************
7767
7768
7769//*************************************************************************************************
7783template< typename MT // Type of the dense matrix
7784 , size_t... CSAs > // Compile time submatrix arguments
7785template< typename MT2 // Type of the right-hand side matrix
7786 , bool SO > // Storage order of the right-hand side matrix
7787inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
7788 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7789{
7793
7794 using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
7795
7797
7798 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
7799 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7800 }
7801
7802 if( !trySchurAssign( matrix_, *rhs, row(), column() ) ) {
7803 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7804 }
7805
7806 decltype(auto) left( derestrict( *this ) );
7807
7808 if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) || (*rhs).canAlias( this ) ) {
7809 const SchurType tmp( *this % (*rhs) );
7810 if( IsSparseMatrix_v<SchurType> )
7811 reset();
7812 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
7813 }
7814 else {
7815 smpSchurAssign( left, transIf< IsSymmetric_v<This> >( *rhs ) );
7816 }
7817
7818 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7819
7820 return *this;
7821}
7823//*************************************************************************************************
7824
7825
7826//*************************************************************************************************
7840template< typename MT // Type of the dense matrix
7841 , size_t... CSAs > // Compile time submatrix arguments
7842template< typename MT2 // Type of the right-hand side matrix
7843 , bool SO > // Storage order of the right-hand side matrix
7844inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
7845 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7846{
7850
7851 using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
7852
7854
7855 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
7856 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7857 }
7858
7859 const SchurType tmp( *this % (*rhs) );
7860
7861 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7862 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7863 }
7864
7865 decltype(auto) left( derestrict( *this ) );
7866
7867 if( IsSparseMatrix_v<SchurType> ) {
7868 reset();
7869 }
7870
7871 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
7872
7873 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7874
7875 return *this;
7876}
7878//*************************************************************************************************
7879
7880
7881
7882
7883//=================================================================================================
7884//
7885// UTILITY FUNCTIONS
7886//
7887//=================================================================================================
7888
7889//*************************************************************************************************
7895template< typename MT // Type of the dense matrix
7896 , size_t... CSAs > // Compile time submatrix arguments
7897inline MT& Submatrix<MT,aligned,false,true,CSAs...>::operand() noexcept
7898{
7899 return matrix_;
7900}
7902//*************************************************************************************************
7903
7904
7905//*************************************************************************************************
7911template< typename MT // Type of the dense matrix
7912 , size_t... CSAs > // Compile time submatrix arguments
7913inline const MT& Submatrix<MT,aligned,false,true,CSAs...>::operand() const noexcept
7914{
7915 return matrix_;
7916}
7918//*************************************************************************************************
7919
7920
7921//*************************************************************************************************
7932template< typename MT // Type of the dense matrix
7933 , size_t... CSAs > // Compile time submatrix arguments
7934inline size_t Submatrix<MT,aligned,false,true,CSAs...>::spacing() const noexcept
7935{
7936 return matrix_.spacing();
7937}
7939//*************************************************************************************************
7940
7941
7942//*************************************************************************************************
7948template< typename MT // Type of the dense matrix
7949 , size_t... CSAs > // Compile time submatrix arguments
7950inline size_t Submatrix<MT,aligned,false,true,CSAs...>::capacity() const noexcept
7951{
7952 return rows() * columns();
7953}
7955//*************************************************************************************************
7956
7957
7958//*************************************************************************************************
7970template< typename MT // Type of the dense matrix
7971 , size_t... CSAs > // Compile time submatrix arguments
7972inline size_t Submatrix<MT,aligned,false,true,CSAs...>::capacity( size_t i ) const noexcept
7973{
7974 MAYBE_UNUSED( i );
7975
7976 BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
7977
7978 return columns();
7979}
7981//*************************************************************************************************
7982
7983
7984//*************************************************************************************************
7990template< typename MT // Type of the dense matrix
7991 , size_t... CSAs > // Compile time submatrix arguments
7993{
7994 const size_t iend( row() + rows() );
7995 const size_t jend( column() + columns() );
7996 size_t nonzeros( 0UL );
7997
7998 for( size_t i=row(); i<iend; ++i )
7999 for( size_t j=column(); j<jend; ++j )
8000 if( !isDefault( matrix_(i,j) ) )
8001 ++nonzeros;
8002
8003 return nonzeros;
8004}
8006//*************************************************************************************************
8007
8008
8009//*************************************************************************************************
8021template< typename MT // Type of the dense matrix
8022 , size_t... CSAs > // Compile time submatrix arguments
8023inline size_t Submatrix<MT,aligned,false,true,CSAs...>::nonZeros( size_t i ) const
8024{
8025 BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
8026
8027 const size_t jend( column() + columns() );
8028 size_t nonzeros( 0UL );
8029
8030 for( size_t j=column(); j<jend; ++j )
8031 if( !isDefault( matrix_(row()+i,j) ) )
8032 ++nonzeros;
8033
8034 return nonzeros;
8035}
8037//*************************************************************************************************
8038
8039
8040//*************************************************************************************************
8046template< typename MT // Type of the dense matrix
8047 , size_t... CSAs > // Compile time submatrix arguments
8049{
8050 using blaze::clear;
8051
8052 for( size_t i=row(); i<row()+rows(); ++i )
8053 {
8054 const size_t jbegin( ( IsUpper_v<MT> )
8055 ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
8056 ?( max( i+1UL, column() ) )
8057 :( max( i, column() ) ) )
8058 :( column() ) );
8059 const size_t jend ( ( IsLower_v<MT> )
8060 ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
8061 ?( min( i, column()+columns() ) )
8062 :( min( i+1UL, column()+columns() ) ) )
8063 :( column()+columns() ) );
8064
8065 for( size_t j=jbegin; j<jend; ++j )
8066 clear( matrix_(i,j) );
8067 }
8068}
8070//*************************************************************************************************
8071
8072
8073//*************************************************************************************************
8085template< typename MT // Type of the dense matrix
8086 , size_t... CSAs > // Compile time submatrix arguments
8088{
8089 using blaze::clear;
8090
8091 BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
8092
8093 const size_t jbegin( ( IsUpper_v<MT> )
8094 ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
8095 ?( max( i+1UL, column() ) )
8096 :( max( i, column() ) ) )
8097 :( column() ) );
8098 const size_t jend ( ( IsLower_v<MT> )
8099 ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
8100 ?( min( i, column()+columns() ) )
8101 :( min( i+1UL, column()+columns() ) ) )
8102 :( column()+columns() ) );
8103
8104 for( size_t j=jbegin; j<jend; ++j )
8105 clear( matrix_(row()+i,j) );
8106}
8108//*************************************************************************************************
8109
8110
8111//*************************************************************************************************
8121template< typename MT // Type of the dense matrix
8122 , size_t... CSAs > // Compile time submatrix arguments
8123inline bool Submatrix<MT,aligned,false,true,CSAs...>::hasOverlap() const noexcept
8124{
8125 BLAZE_INTERNAL_ASSERT( IsSymmetric_v<MT> || IsHermitian_v<MT>, "Invalid matrix detected" );
8126
8127 if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
8128 return false;
8129 else return true;
8130}
8132//*************************************************************************************************
8133
8134
8135
8136
8137//=================================================================================================
8138//
8139// NUMERIC FUNCTIONS
8140//
8141//=================================================================================================
8142
8143//*************************************************************************************************
8161template< typename MT // Type of the dense matrix
8162 , size_t... CSAs > // Compile time submatrix arguments
8163inline Submatrix<MT,aligned,false,true,CSAs...>&
8165{
8166 if( rows() != columns() ) {
8167 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
8168 }
8169
8170 if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
8171 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
8172 }
8173
8174 decltype(auto) left( derestrict( *this ) );
8175 const ResultType tmp( trans( *this ) );
8176
8177 smpAssign( left, tmp );
8178
8179 return *this;
8180}
8182//*************************************************************************************************
8183
8184
8185//*************************************************************************************************
8203template< typename MT // Type of the dense matrix
8204 , size_t... CSAs > // Compile time submatrix arguments
8205inline Submatrix<MT,aligned,false,true,CSAs...>&
8207{
8208 if( rows() != columns() ) {
8209 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
8210 }
8211
8212 if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
8213 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
8214 }
8215
8216 decltype(auto) left( derestrict( *this ) );
8217 const ResultType tmp( ctrans( *this ) );
8218
8219 smpAssign( left, tmp );
8220
8221 return *this;
8222}
8224//*************************************************************************************************
8225
8226
8227//*************************************************************************************************
8240template< typename MT // Type of the dense matrix
8241 , size_t... CSAs > // Compile time submatrix arguments
8242template< typename Other > // Data type of the scalar value
8243inline Submatrix<MT,aligned,false,true,CSAs...>&
8244 Submatrix<MT,aligned,false,true,CSAs...>::scale( const Other& scalar )
8245{
8247
8248 const size_t iend( row() + rows() );
8249
8250 for( size_t i=row(); i<iend; ++i )
8251 {
8252 const size_t jbegin( ( IsUpper_v<MT> )
8253 ?( ( IsStrictlyUpper_v<MT> )
8254 ?( max( i+1UL, column() ) )
8255 :( max( i, column() ) ) )
8256 :( column() ) );
8257 const size_t jend ( ( IsLower_v<MT> )
8258 ?( ( IsStrictlyLower_v<MT> )
8259 ?( min( i, column()+columns() ) )
8260 :( min( i+1UL, column()+columns() ) ) )
8261 :( column()+columns() ) );
8262
8263 for( size_t j=jbegin; j<jend; ++j )
8264 matrix_(i,j) *= scalar;
8265 }
8266
8267 return *this;
8268}
8270//*************************************************************************************************
8271
8272
8273
8274
8275//=================================================================================================
8276//
8277// EXPRESSION TEMPLATE EVALUATION FUNCTIONS
8278//
8279//=================================================================================================
8280
8281//*************************************************************************************************
8292template< typename MT // Type of the dense matrix
8293 , size_t... CSAs > // Compile time submatrix arguments
8294template< typename Other > // Data type of the foreign expression
8295inline bool Submatrix<MT,aligned,false,true,CSAs...>::canAlias( const Other* alias ) const noexcept
8296{
8297 return matrix_.isAliased( &unview( *alias ) );
8298}
8300//*************************************************************************************************
8301
8302
8303//*************************************************************************************************
8314template< typename MT // Type of the dense matrix
8315 , size_t... CSAs > // Compile time submatrix arguments
8316template< typename MT2 // Data type of the foreign dense submatrix
8317 , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
8318 , bool SO2 // Storage order of the foreign dense submatrix
8319 , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
8320inline bool
8321 Submatrix<MT,aligned,false,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
8322{
8323 return ( matrix_.isAliased( &alias->matrix_ ) &&
8324 ( row() + rows() > alias->row() ) &&
8325 ( row() < alias->row() + alias->rows() ) &&
8326 ( column() + columns() > alias->column() ) &&
8327 ( column() < alias->column() + alias->columns() ) );
8328}
8330//*************************************************************************************************
8331
8332
8333//*************************************************************************************************
8344template< typename MT // Type of the dense matrix
8345 , size_t... CSAs > // Compile time submatrix arguments
8346template< typename Other > // Data type of the foreign expression
8347inline bool Submatrix<MT,aligned,false,true,CSAs...>::isAliased( const Other* alias ) const noexcept
8348{
8349 return matrix_.isAliased( &unview( *alias ) );
8350}
8352//*************************************************************************************************
8353
8354
8355//*************************************************************************************************
8366template< typename MT // Type of the dense matrix
8367 , size_t... CSAs > // Compile time submatrix arguments
8368template< typename MT2 // Data type of the foreign dense submatrix
8369 , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
8370 , bool SO2 // Storage order of the foreign dense submatrix
8371 , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
8372inline bool
8373 Submatrix<MT,aligned,false,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
8374{
8375 return ( matrix_.isAliased( &alias->matrix_ ) &&
8376 ( row() + rows() > alias->row() ) &&
8377 ( row() < alias->row() + alias->rows() ) &&
8378 ( column() + columns() > alias->column() ) &&
8379 ( column() < alias->column() + alias->columns() ) );
8380}
8382//*************************************************************************************************
8383
8384
8385//*************************************************************************************************
8395template< typename MT // Type of the dense matrix
8396 , size_t... CSAs > // Compile time submatrix arguments
8397inline bool Submatrix<MT,aligned,false,true,CSAs...>::isAligned() const noexcept
8398{
8399 return true;
8400}
8402//*************************************************************************************************
8403
8404
8405//*************************************************************************************************
8416template< typename MT // Type of the dense matrix
8417 , size_t... CSAs > // Compile time submatrix arguments
8418inline bool Submatrix<MT,aligned,false,true,CSAs...>::canSMPAssign() const noexcept
8419{
8420 return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
8421}
8423//*************************************************************************************************
8424
8425
8426//*************************************************************************************************
8442template< typename MT // Type of the dense matrix
8443 , size_t... CSAs > // Compile time submatrix arguments
8444BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,false,true,CSAs...>::SIMDType
8445 Submatrix<MT,aligned,false,true,CSAs...>::load( size_t i, size_t j ) const noexcept
8446{
8447 return loada( i, j );
8448}
8450//*************************************************************************************************
8451
8452
8453//*************************************************************************************************
8469template< typename MT // Type of the dense matrix
8470 , size_t... CSAs > // Compile time submatrix arguments
8471BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,false,true,CSAs...>::SIMDType
8472 Submatrix<MT,aligned,false,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
8473{
8475
8476 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8477 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8478 BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8479 BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8480
8481 return matrix_.loada( row()+i, column()+j );
8482}
8484//*************************************************************************************************
8485
8486
8487//*************************************************************************************************
8503template< typename MT // Type of the dense matrix
8504 , size_t... CSAs > // Compile time submatrix arguments
8505BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,false,true,CSAs...>::SIMDType
8506 Submatrix<MT,aligned,false,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
8507{
8509
8510 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8511 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8512 BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8513
8514 return matrix_.loadu( row()+i, column()+j );
8515}
8517//*************************************************************************************************
8518
8519
8520//*************************************************************************************************
8537template< typename MT // Type of the dense matrix
8538 , size_t... CSAs > // Compile time submatrix arguments
8540 Submatrix<MT,aligned,false,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
8541{
8542 return storea( i, j, value );
8543}
8545//*************************************************************************************************
8546
8547
8548//*************************************************************************************************
8565template< typename MT // Type of the dense matrix
8566 , size_t... CSAs > // Compile time submatrix arguments
8568 Submatrix<MT,aligned,false,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
8569{
8571
8572 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8573 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8574 BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8575 BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8576
8577 return matrix_.storea( row()+i, column()+j, value );
8578}
8580//*************************************************************************************************
8581
8582
8583//*************************************************************************************************
8600template< typename MT // Type of the dense matrix
8601 , size_t... CSAs > // Compile time submatrix arguments
8603 Submatrix<MT,aligned,false,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
8604{
8606
8607 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8608 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8609 BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8610
8611 matrix_.storeu( row()+i, column()+j, value );
8612}
8614//*************************************************************************************************
8615
8616
8617//*************************************************************************************************
8635template< typename MT // Type of the dense matrix
8636 , size_t... CSAs > // Compile time submatrix arguments
8638 Submatrix<MT,aligned,false,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
8639{
8641
8642 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8643 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8644 BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8645 BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8646
8647 matrix_.stream( row()+i, column()+j, value );
8648}
8650//*************************************************************************************************
8651
8652
8653//*************************************************************************************************
8665template< typename MT // Type of the dense matrix
8666 , size_t... CSAs > // Compile time submatrix arguments
8667template< typename MT2 > // Type of the right-hand side dense matrix
8668inline auto Submatrix<MT,aligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
8669 -> DisableIf_t< VectorizedAssign_v<MT2> >
8670{
8671 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
8672 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
8673
8674 const size_t jpos( prevMultiple( columns(), 2UL ) );
8675 BLAZE_INTERNAL_ASSERT( jpos <= columns(), "Invalid end calculation" );
8676
8677 for( size_t i=0UL; i<rows(); ++i ) {
8678 for( size_t j=0UL; j<jpos; j+=2UL ) {
8679 matrix_(row()+i,column()+j ) = (*rhs)(i,j );
8680 matrix_(row()+i,column()+j+1UL) = (*rhs)(i,j+1UL);
8681 }
8682 if( jpos < columns() ) {
8683 matrix_(row()+i,column()+jpos) = (*rhs)(i,jpos);
8684 }
8685 }
8686}
8688//*************************************************************************************************
8689
8690
8691//*************************************************************************************************
8703template< typename MT // Type of the dense matrix
8704 , size_t... CSAs > // Compile time submatrix arguments
8705template< typename MT2 > // Type of the right-hand side dense matrix
8706inline auto Submatrix<MT,aligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
8707 -> EnableIf_t< VectorizedAssign_v<MT2> >
8708{
8710
8711 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
8712 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
8713
8714 const size_t jpos( prevMultiple( columns(), SIMDSIZE ) );
8715 BLAZE_INTERNAL_ASSERT( jpos <= columns(), "Invalid end calculation" );
8716
8717 if( useStreaming &&
8718 rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
8719 !(*rhs).isAliased( this ) )
8720 {
8721 for( size_t i=0UL; i<rows(); ++i )
8722 {
8723 size_t j( 0UL );
8724 Iterator left( begin(i) );
8725 ConstIterator_t<MT2> right( (*rhs).begin(i) );
8726
8727 for( ; j<jpos; j+=SIMDSIZE ) {
8728 left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8729 }
8730 for( ; j<columns(); ++j ) {
8731 *left = *right; ++left; ++right;
8732 }
8733 }
8734 }
8735 else
8736 {
8737 for( size_t i=0UL; i<rows(); ++i )
8738 {
8739 size_t j( 0UL );
8740 Iterator left( begin(i) );
8741 ConstIterator_t<MT2> right( (*rhs).begin(i) );
8742
8743 for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
8744 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8745 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8746 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8747 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8748 }
8749 for( ; j<jpos; j+=SIMDSIZE ) {
8750 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8751 }
8752 for( ; j<columns(); ++j ) {
8753 *left = *right; ++left; ++right;
8754 }
8755 }
8756 }
8757}
8759//*************************************************************************************************
8760
8761
8762//*************************************************************************************************
8774template< typename MT // Type of the dense matrix
8775 , size_t... CSAs > // Compile time submatrix arguments
8776template< typename MT2 > // Type of the right-hand side dense matrix
8777inline void Submatrix<MT,aligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
8778{
8780
8781 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
8782 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
8783
8784 constexpr size_t block( BLOCK_SIZE );
8785
8786 for( size_t ii=0UL; ii<rows(); ii+=block ) {
8787 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
8788 for( size_t jj=0UL; jj<columns(); jj+=block ) {
8789 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
8790 for( size_t i=ii; i<iend; ++i ) {
8791 for( size_t j=jj; j<jend; ++j ) {
8792 matrix_(row()+i,column()+j) = (*rhs)(i,j);
8793 }
8794 }
8795 }
8796 }
8797}
8799//*************************************************************************************************
8800
8801
8802//*************************************************************************************************
8814template< typename MT // Type of the dense matrix
8815 , size_t... CSAs > // Compile time submatrix arguments
8816template< typename MT2 > // Type of the right-hand side sparse matrix
8817inline void Submatrix<MT,aligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
8818{
8819 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
8820 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
8821
8822 for( size_t i=0UL; i<rows(); ++i )
8823 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element )
8824 matrix_(row()+i,column()+element->index()) = element->value();
8825}
8827//*************************************************************************************************
8828
8829
8830//*************************************************************************************************
8842template< typename MT // Type of the dense matrix
8843 , size_t... CSAs > // Compile time submatrix arguments
8844template< typename MT2 > // Type of the right-hand side sparse matrix
8845inline void Submatrix<MT,aligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
8846{
8848
8849 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
8850 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
8851
8852 for( size_t j=0UL; j<columns(); ++j )
8853 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element )
8854 matrix_(row()+element->index(),column()+j) = element->value();
8855}
8857//*************************************************************************************************
8858
8859
8860//*************************************************************************************************
8872template< typename MT // Type of the dense matrix
8873 , size_t... CSAs > // Compile time submatrix arguments
8874template< typename MT2 > // Type of the right-hand side dense matrix
8875inline auto Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
8876 -> DisableIf_t< VectorizedAddAssign_v<MT2> >
8877{
8878 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
8879 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
8880
8881 const size_t jpos( prevMultiple( columns(), 2UL ) );
8882 BLAZE_INTERNAL_ASSERT( jpos <= columns(), "Invalid end calculation" );
8883
8884 for( size_t i=0UL; i<rows(); ++i )
8885 {
8886 if( IsDiagonal_v<MT2> ) {
8887 matrix_(row()+i,column()+i) += (*rhs)(i,i);
8888 }
8889 else {
8890 for( size_t j=0UL; j<jpos; j+=2UL ) {
8891 matrix_(row()+i,column()+j ) += (*rhs)(i,j );
8892 matrix_(row()+i,column()+j+1UL) += (*rhs)(i,j+1UL);
8893 }
8894 if( jpos < columns() ) {
8895 matrix_(row()+i,column()+jpos) += (*rhs)(i,jpos);
8896 }
8897 }
8898 }
8899}
8901//*************************************************************************************************
8902
8903
8904//*************************************************************************************************
8916template< typename MT // Type of the dense matrix
8917 , size_t... CSAs > // Compile time submatrix arguments
8918template< typename MT2 > // Type of the right-hand side dense matrix
8919inline auto Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
8920 -> EnableIf_t< VectorizedAddAssign_v<MT2> >
8921{
8923
8924 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
8925 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
8926
8927 for( size_t i=0UL; i<rows(); ++i )
8928 {
8929 const size_t jbegin( ( IsUpper_v<MT2> )
8930 ?( prevMultiple( ( IsStrictlyUpper_v<MT2> ? i+1UL : i ), SIMDSIZE ) )
8931 :( 0UL ) );
8932 const size_t jend ( ( IsLower_v<MT2> )
8933 ?( IsStrictlyLower_v<MT2> ? i : i+1UL )
8934 :( columns() ) );
8935 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
8936
8937 const size_t jpos( prevMultiple( jend, SIMDSIZE ) );
8938 BLAZE_INTERNAL_ASSERT( jpos <= jend, "Invalid end calculation" );
8939
8940 size_t j( jbegin );
8941 Iterator left( begin(i) + jbegin );
8942 ConstIterator_t<MT2> right( (*rhs).begin(i) + jbegin );
8943
8944 for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
8945 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8946 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8947 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8948 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8949 }
8950 for( ; j<jpos; j+=SIMDSIZE ) {
8951 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8952 }
8953 for( ; j<jend; ++j ) {
8954 *left += *right; ++left; ++right;
8955 }
8956 }
8957}
8959//*************************************************************************************************
8960
8961
8962//*************************************************************************************************
8974template< typename MT // Type of the dense matrix
8975 , size_t... CSAs > // Compile time submatrix arguments
8976template< typename MT2 > // Type of the right-hand side dense matrix
8977inline void Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
8978{
8980
8981 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
8982 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
8983
8984 constexpr size_t block( BLOCK_SIZE );
8985
8986 for( size_t ii=0UL; ii<rows(); ii+=block ) {
8987 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
8988 for( size_t jj=0UL; jj<columns(); jj+=block ) {
8989 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
8990 for( size_t i=ii; i<iend; ++i ) {
8991 for( size_t j=jj; j<jend; ++j ) {
8992 matrix_(row()+i,column()+j) += (*rhs)(i,j);
8993 }
8994 }
8995 }
8996 }
8997}
8999//*************************************************************************************************
9000
9001
9002//*************************************************************************************************
9014template< typename MT // Type of the dense matrix
9015 , size_t... CSAs > // Compile time submatrix arguments
9016template< typename MT2 > // Type of the right-hand side sparse matrix
9017inline void Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
9018{
9019 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
9020 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
9021
9022 for( size_t i=0UL; i<rows(); ++i )
9023 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element )
9024 matrix_(row()+i,column()+element->index()) += element->value();
9025}
9027//*************************************************************************************************
9028
9029
9030//*************************************************************************************************
9042template< typename MT // Type of the dense matrix
9043 , size_t... CSAs > // Compile time submatrix arguments
9044template< typename MT2 > // Type of the right-hand side sparse matrix
9045inline void Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
9046{
9048
9049 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
9050 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
9051
9052 for( size_t j=0UL; j<columns(); ++j )
9053 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element )
9054 matrix_(row()+element->index(),column()+j) += element->value();
9055}
9057//*************************************************************************************************
9058
9059
9060//*************************************************************************************************
9072template< typename MT // Type of the dense matrix
9073 , size_t... CSAs > // Compile time submatrix arguments
9074template< typename MT2 > // Type of the right-hand side dense matrix
9075inline auto Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
9076 -> DisableIf_t< VectorizedSubAssign_v<MT2> >
9077{
9078 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
9079 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
9080
9081 const size_t jpos( prevMultiple( columns(), 2UL ) );
9082 BLAZE_INTERNAL_ASSERT( jpos <= columns(), "Invalid end calculation" );
9083
9084 for( size_t i=0UL; i<rows(); ++i )
9085 {
9086 if( IsDiagonal_v<MT2> ) {
9087 matrix_(row()+i,column()+i) -= (*rhs)(i,i);
9088 }
9089 else {
9090 for( size_t j=0UL; j<jpos; j+=2UL ) {
9091 matrix_(row()+i,column()+j ) -= (*rhs)(i,j );
9092 matrix_(row()+i,column()+j+1UL) -= (*rhs)(i,j+1UL);
9093 }
9094 if( jpos < columns() ) {
9095 matrix_(row()+i,column()+jpos) -= (*rhs)(i,jpos);
9096 }
9097 }
9098 }
9099}
9101//*************************************************************************************************
9102
9103
9104//*************************************************************************************************
9116template< typename MT // Type of the dense matrix
9117 , size_t... CSAs > // Compile time submatrix arguments
9118template< typename MT2 > // Type of the right-hand side dense matrix
9119inline auto Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
9120 -> EnableIf_t< VectorizedSubAssign_v<MT2> >
9121{
9123
9124 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
9125 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
9126
9127 for( size_t i=0UL; i<rows(); ++i )
9128 {
9129 const size_t jbegin( ( IsUpper_v<MT2> )
9130 ?( prevMultiple( ( IsStrictlyUpper_v<MT2> ? i+1UL : i ), SIMDSIZE ) )
9131 :( 0UL ) );
9132 const size_t jend ( ( IsLower_v<MT2> )
9133 ?( IsStrictlyLower_v<MT2> ? i : i+1UL )
9134 :( columns() ) );
9135 BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
9136
9137 const size_t jpos( prevMultiple( jend, SIMDSIZE ) );
9138 BLAZE_INTERNAL_ASSERT( jpos <= jend, "Invalid end calculation" );
9139
9140 size_t j( jbegin );
9141 Iterator left( begin(i) + jbegin );
9142 ConstIterator_t<MT2> right( (*rhs).begin(i) + jbegin );
9143
9144 for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
9145 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9146 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9147 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9148 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9149 }
9150 for( ; j<jpos; j+=SIMDSIZE ) {
9151 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9152 }
9153 for( ; j<jend; ++j ) {
9154 *left -= *right; ++left; ++right;
9155 }
9156 }
9157}
9159//*************************************************************************************************
9160
9161
9162//*************************************************************************************************
9174template< typename MT // Type of the dense matrix
9175 , size_t... CSAs > // Compile time submatrix arguments
9176template< typename MT2 > // Type of the right-hand side dense matrix
9177inline void Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
9178{
9180
9181 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
9182 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
9183
9184 constexpr size_t block( BLOCK_SIZE );
9185
9186 for( size_t ii=0UL; ii<rows(); ii+=block ) {
9187 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
9188 for( size_t jj=0UL; jj<columns(); jj+=block ) {
9189 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
9190 for( size_t i=ii; i<iend; ++i ) {
9191 for( size_t j=jj; j<jend; ++j ) {
9192 matrix_(row()+i,column()+j) -= (*rhs)(i,j);
9193 }
9194 }
9195 }
9196 }
9197}
9199//*************************************************************************************************
9200
9201
9202//*************************************************************************************************
9214template< typename MT // Type of the dense matrix
9215 , size_t... CSAs > // Compile time submatrix arguments
9216template< typename MT2 > // Type of the right-hand side sparse matrix
9217inline void Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
9218{
9219 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
9220 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
9221
9222 for( size_t i=0UL; i<rows(); ++i )
9223 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element )
9224 matrix_(row()+i,column()+element->index()) -= element->value();
9225}
9227//*************************************************************************************************
9228
9229
9230//*************************************************************************************************
9242template< typename MT // Type of the dense matrix
9243 , size_t... CSAs > // Compile time submatrix arguments
9244template< typename MT2 > // Type of the right-hand side sparse matrix
9245inline void Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
9246{
9248
9249 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
9250 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
9251
9252 for( size_t j=0UL; j<columns(); ++j )
9253 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element )
9254 matrix_(row()+element->index(),column()+j) -= element->value();
9255}
9257//*************************************************************************************************
9258
9259
9260//*************************************************************************************************
9272template< typename MT // Type of the dense matrix
9273 , size_t... CSAs > // Compile time submatrix arguments
9274template< typename MT2 > // Type of the right-hand side dense matrix
9275inline auto Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
9276 -> DisableIf_t< VectorizedSchurAssign_v<MT2> >
9277{
9278 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
9279 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
9280
9281 const size_t jpos( prevMultiple( columns(), 2UL ) );
9282 BLAZE_INTERNAL_ASSERT( jpos <= columns(), "Invalid end calculation" );
9283
9284 for( size_t i=0UL; i<rows(); ++i ) {
9285 for( size_t j=0UL; j<jpos; j+=2UL ) {
9286 matrix_(row()+i,column()+j ) *= (*rhs)(i,j );
9287 matrix_(row()+i,column()+j+1UL) *= (*rhs)(i,j+1UL);
9288 }
9289 if( jpos < columns() ) {
9290 matrix_(row()+i,column()+jpos) *= (*rhs)(i,jpos);
9291 }
9292 }
9293}
9295//*************************************************************************************************
9296
9297
9298//*************************************************************************************************
9310template< typename MT // Type of the dense matrix
9311 , size_t... CSAs > // Compile time submatrix arguments
9312template< typename MT2 > // Type of the right-hand side dense matrix
9313inline auto Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
9314 -> EnableIf_t< VectorizedSchurAssign_v<MT2> >
9315{
9317
9318 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
9319 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
9320
9321 for( size_t i=0UL; i<rows(); ++i )
9322 {
9323 const size_t jpos( prevMultiple( columns(), SIMDSIZE ) );
9324 BLAZE_INTERNAL_ASSERT( jpos <= columns(), "Invalid end calculation" );
9325
9326 size_t j( 0UL );
9327 Iterator left( begin(i) );
9328 ConstIterator_t<MT2> right( (*rhs).begin(i) );
9329
9330 for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
9331 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9332 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9333 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9334 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9335 }
9336 for( ; j<jpos; j+=SIMDSIZE ) {
9337 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9338 }
9339 for( ; j<columns(); ++j ) {
9340 *left *= *right; ++left; ++right;
9341 }
9342 }
9343}
9345//*************************************************************************************************
9346
9347
9348//*************************************************************************************************
9360template< typename MT // Type of the dense matrix
9361 , size_t... CSAs > // Compile time submatrix arguments
9362template< typename MT2 > // Type of the right-hand side dense matrix
9363inline void Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
9364{
9366
9367 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
9368 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
9369
9370 constexpr size_t block( BLOCK_SIZE );
9371
9372 for( size_t ii=0UL; ii<rows(); ii+=block ) {
9373 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
9374 for( size_t jj=0UL; jj<columns(); jj+=block ) {
9375 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
9376 for( size_t i=ii; i<iend; ++i ) {
9377 for( size_t j=jj; j<jend; ++j ) {
9378 matrix_(row()+i,column()+j) *= (*rhs)(i,j);
9379 }
9380 }
9381 }
9382 }
9383}
9385//*************************************************************************************************
9386
9387
9388//*************************************************************************************************
9400template< typename MT // Type of the dense matrix
9401 , size_t... CSAs > // Compile time submatrix arguments
9402template< typename MT2 > // Type of the right-hand side sparse matrix
9403inline void Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
9404{
9405 using blaze::reset;
9406
9407 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
9408 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
9409
9410 for( size_t i=0UL; i<rows(); ++i )
9411 {
9412 size_t j( 0UL );
9413
9414 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element ) {
9415 for( ; j<element->index(); ++j )
9416 reset( matrix_(row()+i,column()+j) );
9417 matrix_(row()+i,column()+j) *= element->value();
9418 ++j;
9419 }
9420
9421 for( ; j<columns(); ++j ) {
9422 reset( matrix_(row()+i,column()+j) );
9423 }
9424 }
9425}
9427//*************************************************************************************************
9428
9429
9430//*************************************************************************************************
9442template< typename MT // Type of the dense matrix
9443 , size_t... CSAs > // Compile time submatrix arguments
9444template< typename MT2 > // Type of the right-hand side sparse matrix
9445inline void Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
9446{
9447 using blaze::reset;
9448
9450
9451 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
9452 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
9453
9454 for( size_t j=0UL; j<columns(); ++j )
9455 {
9456 size_t i( 0UL );
9457
9458 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element ) {
9459 for( ; i<element->index(); ++i )
9460 reset( matrix_(row()+i,column()+j) );
9461 matrix_(row()+element->index(),column()+j) *= element->value();
9462 ++i;
9463 }
9464
9465 for( ; i<rows(); ++i ) {
9466 reset( matrix_(row()+i,column()+j) );
9467 }
9468 }
9469}
9471//*************************************************************************************************
9472
9473
9474
9475
9476
9477
9478
9479
9480//=================================================================================================
9481//
9482// CLASS TEMPLATE SPECIALIZATION FOR ALIGNED COLUMN-MAJOR DENSE SUBMATRICES
9483//
9484//=================================================================================================
9485
9486//*************************************************************************************************
9494template< typename MT // Type of the dense matrix
9495 , size_t... CSAs > // Compile time submatrix arguments
9496class Submatrix<MT,aligned,true,true,CSAs...>
9497 : public View< DenseMatrix< Submatrix<MT,aligned,true,true,CSAs...>, true > >
9498 , private SubmatrixData<CSAs...>
9499{
9500 private:
9501 //**Type definitions****************************************************************************
9502 using DataType = SubmatrixData<CSAs...>;
9503 using Operand = If_t< IsExpression_v<MT>, MT, MT& >;
9504 //**********************************************************************************************
9505
9506 //**********************************************************************************************
9508 template< typename MT1, typename MT2 >
9509 static constexpr bool EnforceEvaluation_v =
9510 ( IsRestricted_v<MT1> && RequiresEvaluation_v<MT2> );
9511 //**********************************************************************************************
9512
9513 public:
9514 //**Type definitions****************************************************************************
9516 using This = Submatrix<MT,aligned,true,true,CSAs...>;
9517
9519 using BaseType = View< DenseMatrix<This,true> >;
9520
9521 using ViewedType = MT;
9522 using ResultType = SubmatrixTrait_t<MT,CSAs...>;
9523 using OppositeType = OppositeType_t<ResultType>;
9524 using TransposeType = TransposeType_t<ResultType>;
9525 using ElementType = ElementType_t<MT>;
9526 using SIMDType = SIMDTrait_t<ElementType>;
9527 using ReturnType = ReturnType_t<MT>;
9528 using CompositeType = const Submatrix&;
9529
9531 using ConstReference = ConstReference_t<MT>;
9532
9534 using Reference = If_t< IsConst_v<MT>, ConstReference, Reference_t<MT> >;
9535
9537 using ConstPointer = ConstPointer_t<MT>;
9538
9540 using Pointer = If_t< IsConst_v<MT> || !HasMutableDataAccess_v<MT>, ConstPointer, Pointer_t<MT> >;
9541
9543 using ConstIterator = ConstIterator_t<MT>;
9544
9546 using Iterator = If_t< IsConst_v<MT>, ConstIterator, Iterator_t<MT> >;
9547 //**********************************************************************************************
9548
9549 //**Compilation flags***************************************************************************
9551 static constexpr bool simdEnabled = MT::simdEnabled;
9552
9554 static constexpr bool smpAssignable = MT::smpAssignable;
9555
9557 static constexpr bool compileTimeArgs = DataType::compileTimeArgs;
9558 //**********************************************************************************************
9559
9560 //**Constructors********************************************************************************
9563 template< typename... RSAs >
9564 explicit inline Submatrix( MT& matrix, RSAs... args );
9565
9566 Submatrix( const Submatrix& ) = default;
9568 //**********************************************************************************************
9569
9570 //**Destructor**********************************************************************************
9573 ~Submatrix() = default;
9575 //**********************************************************************************************
9576
9577 //**Data access functions***********************************************************************
9580 inline Reference operator()( size_t i, size_t j );
9581 inline ConstReference operator()( size_t i, size_t j ) const;
9582 inline Reference at( size_t i, size_t j );
9583 inline ConstReference at( size_t i, size_t j ) const;
9584 inline Pointer data () noexcept;
9585 inline ConstPointer data () const noexcept;
9586 inline Pointer data ( size_t j ) noexcept;
9587 inline ConstPointer data ( size_t j ) const noexcept;
9588 inline Iterator begin ( size_t j );
9589 inline ConstIterator begin ( size_t j ) const;
9590 inline ConstIterator cbegin( size_t j ) const;
9591 inline Iterator end ( size_t j );
9592 inline ConstIterator end ( size_t j ) const;
9593 inline ConstIterator cend ( size_t j ) const;
9595 //**********************************************************************************************
9596
9597 //**Assignment operators************************************************************************
9600 inline Submatrix& operator=( const ElementType& rhs );
9601 inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
9602 inline Submatrix& operator=( const Submatrix& rhs );
9603
9604 template< typename MT2, bool SO >
9605 inline Submatrix& operator=( const Matrix<MT2,SO>& rhs );
9606
9607 template< typename MT2, bool SO >
9608 inline auto operator+=( const Matrix<MT2,SO>& rhs )
9609 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9610
9611 template< typename MT2, bool SO >
9612 inline auto operator+=( const Matrix<MT2,SO>& rhs )
9613 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9614
9615 template< typename MT2, bool SO >
9616 inline auto operator-=( const Matrix<MT2,SO>& rhs )
9617 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9618
9619 template< typename MT2, bool SO >
9620 inline auto operator-=( const Matrix<MT2,SO>& rhs )
9621 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9622
9623 template< typename MT2, bool SO >
9624 inline auto operator%=( const Matrix<MT2,SO>& rhs )
9625 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9626
9627 template< typename MT2, bool SO >
9628 inline auto operator%=( const Matrix<MT2,SO>& rhs )
9629 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9631 //**********************************************************************************************
9632
9633 //**Utility functions***************************************************************************
9636 using DataType::row;
9637 using DataType::column;
9638 using DataType::rows;
9639 using DataType::columns;
9640
9641 inline MT& operand() noexcept;
9642 inline const MT& operand() const noexcept;
9643
9644 inline size_t spacing() const noexcept;
9645 inline size_t capacity() const noexcept;
9646 inline size_t capacity( size_t i ) const noexcept;
9647 inline size_t nonZeros() const;
9648 inline size_t nonZeros( size_t i ) const;
9649 inline void reset();
9650 inline void reset( size_t i );
9652 //**********************************************************************************************
9653
9654 //**Numeric functions***************************************************************************
9657 inline Submatrix& transpose();
9658 inline Submatrix& ctranspose();
9659
9660 template< typename Other > inline Submatrix& scale( const Other& scalar );
9662 //**********************************************************************************************
9663
9664 private:
9665 //**********************************************************************************************
9667 template< typename MT2 >
9668 static constexpr bool VectorizedAssign_v =
9669 ( useOptimizedKernels &&
9670 simdEnabled && MT2::simdEnabled &&
9671 IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > );
9672 //**********************************************************************************************
9673
9674 //**********************************************************************************************
9676 template< typename MT2 >
9677 static constexpr bool VectorizedAddAssign_v =
9678 ( VectorizedAssign_v<MT2> &&
9679 HasSIMDAdd_v< ElementType, ElementType_t<MT2> > &&
9680 !IsDiagonal_v<MT2> );
9681 //**********************************************************************************************
9682
9683 //**********************************************************************************************
9685 template< typename MT2 >
9686 static constexpr bool VectorizedSubAssign_v =
9687 ( VectorizedAssign_v<MT2> &&
9688 HasSIMDSub_v< ElementType, ElementType_t<MT2> > &&
9689 !IsDiagonal_v<MT2> );
9690 //**********************************************************************************************
9691
9692 //**********************************************************************************************
9694 template< typename MT2 >
9695 static constexpr bool VectorizedSchurAssign_v =
9696 ( VectorizedAssign_v<MT2> &&
9697 HasSIMDMult_v< ElementType, ElementType_t<MT2> > );
9698 //**********************************************************************************************
9699
9700 //**SIMD properties*****************************************************************************
9702 static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
9703 //**********************************************************************************************
9704
9705 public:
9706 //**Expression template evaluation functions****************************************************
9709 template< typename Other >
9710 inline bool canAlias( const Other* alias ) const noexcept;
9711
9712 template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
9713 inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
9714
9715 template< typename Other >
9716 inline bool isAliased( const Other* alias ) const noexcept;
9717
9718 template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
9719 inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
9720
9721 inline bool isAligned () const noexcept;
9722 inline bool canSMPAssign() const noexcept;
9723
9724 BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
9725 BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
9726 BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
9727
9728 BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
9729 BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
9730 BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
9731 BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
9732
9733 template< typename MT2 >
9734 inline auto assign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT2> >;
9735
9736 template< typename MT2 >
9737 inline auto assign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT2> >;
9738
9739 template< typename MT2 > inline void assign( const DenseMatrix<MT2,false>& rhs );
9740 template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
9741 template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
9742
9743 template< typename MT2 >
9744 inline auto addAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT2> >;
9745
9746 template< typename MT2 >
9747 inline auto addAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT2> >;
9748
9749 template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,false>& rhs );
9750 template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
9751 template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
9752
9753 template< typename MT2 >
9754 inline auto subAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT2> >;
9755
9756 template< typename MT2 >
9757 inline auto subAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT2> >;
9758
9759 template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,false>& rhs );
9760 template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
9761 template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
9762
9763 template< typename MT2 >
9764 inline auto schurAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT2> >;
9765
9766 template< typename MT2 >
9767 inline auto schurAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT2> >;
9768
9769 template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,false>& rhs );
9770 template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
9771 template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
9773 //**********************************************************************************************
9774
9775 private:
9776 //**Utility functions***************************************************************************
9779 inline bool hasOverlap() const noexcept;
9781 //**********************************************************************************************
9782
9783 //**Member variables****************************************************************************
9786 Operand matrix_;
9788 //**********************************************************************************************
9789
9790 //**Friend declarations*************************************************************************
9791 template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
9792 //**********************************************************************************************
9793
9794 //**Compile time checks*************************************************************************
9802 //**********************************************************************************************
9803};
9805//*************************************************************************************************
9806
9807
9808
9809
9810//=================================================================================================
9811//
9812// CONSTRUCTORS
9813//
9814//=================================================================================================
9815
9816//*************************************************************************************************
9829template< typename MT // Type of the dense matrix
9830 , size_t... CSAs > // Compile time submatrix arguments
9831template< typename... RSAs > // Runtime submatrix arguments
9832inline Submatrix<MT,aligned,true,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
9833 : DataType( args... ) // Base class initialization
9834 , matrix_ ( matrix ) // The matrix containing the submatrix
9835{
9836 if( isChecked( args... ) )
9837 {
9838 if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
9839 BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
9840 }
9841
9842 if( simdEnabled && IsContiguous_v<MT> &&
9843 ( !checkAlignment( data() ) ||
9844 ( columns() > 1UL && matrix_.spacing() % SIMDSIZE != 0UL ) ) ) {
9845 BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix alignment" );
9846 }
9847 }
9848 else
9849 {
9850 BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
9851 BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
9852
9853 BLAZE_USER_ASSERT( !simdEnabled || !IsContiguous_v<MT> || checkAlignment( data() ), "Invalid submatrix alignment" );
9854 BLAZE_USER_ASSERT( !simdEnabled || !IsContiguous_v<MT> || columns() <= 1UL || matrix_.spacing() % SIMDSIZE == 0UL, "Invalid submatrix alignment" );
9855 }
9856}
9858//*************************************************************************************************
9859
9860
9861
9862
9863//=================================================================================================
9864//
9865// DATA ACCESS FUNCTIONS
9866//
9867//=================================================================================================
9868
9869//*************************************************************************************************
9880template< typename MT // Type of the dense matrix
9881 , size_t... CSAs > // Compile time submatrix arguments
9882inline typename Submatrix<MT,aligned,true,true,CSAs...>::Reference
9883 Submatrix<MT,aligned,true,true,CSAs...>::operator()( size_t i, size_t j )
9884{
9885 BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
9886 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
9887
9888 return matrix_(row()+i,column()+j);
9889}
9891//*************************************************************************************************
9892
9893
9894//*************************************************************************************************
9905template< typename MT // Type of the dense matrix
9906 , size_t... CSAs > // Compile time submatrix arguments
9907inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstReference
9908 Submatrix<MT,aligned,true,true,CSAs...>::operator()( size_t i, size_t j ) const
9909{
9910 BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
9911 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
9912
9913 return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
9914}
9916//*************************************************************************************************
9917
9918
9919//*************************************************************************************************
9931template< typename MT // Type of the dense matrix
9932 , size_t... CSAs > // Compile time submatrix arguments
9933inline typename Submatrix<MT,aligned,true,true,CSAs...>::Reference
9934 Submatrix<MT,aligned,true,true,CSAs...>::at( size_t i, size_t j )
9935{
9936 if( i >= rows() ) {
9937 BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
9938 }
9939 if( j >= columns() ) {
9940 BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
9941 }
9942 return (*this)(i,j);
9943}
9945//*************************************************************************************************
9946
9947
9948//*************************************************************************************************
9960template< typename MT // Type of the dense matrix
9961 , size_t... CSAs > // Compile time submatrix arguments
9962inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstReference
9963 Submatrix<MT,aligned,true,true,CSAs...>::at( size_t i, size_t j ) const
9964{
9965 if( i >= rows() ) {
9966 BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
9967 }
9968 if( j >= columns() ) {
9969 BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
9970 }
9971 return (*this)(i,j);
9972}
9974//*************************************************************************************************
9975
9976
9977//*************************************************************************************************
9987template< typename MT // Type of the dense matrix
9988 , size_t... CSAs > // Compile time submatrix arguments
9989inline typename Submatrix<MT,aligned,true,true,CSAs...>::Pointer
9991{
9992 return matrix_.data() + row() + column()*spacing();
9993}
9995//*************************************************************************************************
9996
9997
9998//*************************************************************************************************
10008template< typename MT // Type of the dense matrix
10009 , size_t... CSAs > // Compile time submatrix arguments
10010inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstPointer
10012{
10013 return matrix_.data() + row() + column()*spacing();
10014}
10016//*************************************************************************************************
10017
10018
10019//*************************************************************************************************
10028template< typename MT // Type of the dense matrix
10029 , size_t... CSAs > // Compile time submatrix arguments
10030inline typename Submatrix<MT,aligned,true,true,CSAs...>::Pointer
10032{
10033 return matrix_.data() + row() + (column()+j)*spacing();
10034}
10036//*************************************************************************************************
10037
10038
10039//*************************************************************************************************
10048template< typename MT // Type of the dense matrix
10049 , size_t... CSAs > // Compile time submatrix arguments
10050inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstPointer
10051 Submatrix<MT,aligned,true,true,CSAs...>::data( size_t j ) const noexcept
10052{
10053 return matrix_.data() + row() + (column()+j)*spacing();
10054}
10056//*************************************************************************************************
10057
10058
10059//*************************************************************************************************
10066template< typename MT // Type of the dense matrix
10067 , size_t... CSAs > // Compile time submatrix arguments
10068inline typename Submatrix<MT,aligned,true,true,CSAs...>::Iterator
10070{
10071 BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10072 return ( matrix_.begin( column() + j ) + row() );
10073}
10075//*************************************************************************************************
10076
10077
10078//*************************************************************************************************
10085template< typename MT // Type of the dense matrix
10086 , size_t... CSAs > // Compile time submatrix arguments
10087inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10089{
10090 BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10091 return ( matrix_.cbegin( column() + j ) + row() );
10092}
10094//*************************************************************************************************
10095
10096
10097//*************************************************************************************************
10104template< typename MT // Type of the dense matrix
10105 , size_t... CSAs > // Compile time submatrix arguments
10106inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10108{
10109 BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10110 return ( matrix_.cbegin( column() + j ) + row() );
10111}
10113//*************************************************************************************************
10114
10115
10116//*************************************************************************************************
10123template< typename MT // Type of the dense matrix
10124 , size_t... CSAs > // Compile time submatrix arguments
10125inline typename Submatrix<MT,aligned,true,true,CSAs...>::Iterator
10127{
10128 BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10129 return ( matrix_.begin( column() + j ) + row() + rows() );
10130}
10132//*************************************************************************************************
10133
10134
10135//*************************************************************************************************
10142template< typename MT // Type of the dense matrix
10143 , size_t... CSAs > // Compile time submatrix arguments
10144inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10146{
10147 BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10148 return ( matrix_.cbegin( column() + j ) + row() + rows() );
10149}
10151//*************************************************************************************************
10152
10153
10154//*************************************************************************************************
10161template< typename MT // Type of the dense matrix
10162 , size_t... CSAs > // Compile time submatrix arguments
10163inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10165{
10166 BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10167 return ( matrix_.cbegin( column() + j ) + row() + rows() );
10168}
10170//*************************************************************************************************
10171
10172
10173
10174
10175//=================================================================================================
10176//
10177// ASSIGNMENT OPERATORS
10178//
10179//=================================================================================================
10180
10181//*************************************************************************************************
10192template< typename MT // Type of the dense matrix
10193 , size_t... CSAs > // Compile time submatrix arguments
10194inline Submatrix<MT,aligned,true,true,CSAs...>&
10195 Submatrix<MT,aligned,true,true,CSAs...>::operator=( const ElementType& rhs )
10196{
10197 const size_t jend( column() + columns() );
10198 decltype(auto) left( derestrict( matrix_ ) );
10199
10200 for( size_t j=column(); j<jend; ++j )
10201 {
10202 const size_t ibegin( ( IsLower_v<MT> )
10203 ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
10204 ?( max( j+1UL, row() ) )
10205 :( max( j, row() ) ) )
10206 :( row() ) );
10207 const size_t iend ( ( IsUpper_v<MT> )
10208 ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
10209 ?( min( j, row()+rows() ) )
10210 :( min( j+1UL, row()+rows() ) ) )
10211 :( row()+rows() ) );
10212
10213 for( size_t i=ibegin; i<iend; ++i ) {
10214 if( !IsRestricted_v<MT> || IsTriangular_v<MT> || trySet( matrix_, i, j, rhs ) )
10215 left(i,j) = rhs;
10216 }
10217 }
10218
10219 return *this;
10220}
10222//*************************************************************************************************
10223
10224
10225//*************************************************************************************************
10241template< typename MT // Type of the dense matrix
10242 , size_t... CSAs > // Compile time submatrix arguments
10243inline Submatrix<MT,aligned,true,true,CSAs...>&
10244 Submatrix<MT,aligned,true,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
10245{
10246 using blaze::reset;
10247
10248 if( list.size() != rows() ) {
10249 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
10250 }
10251
10252 if( IsRestricted_v<MT> ) {
10253 const InitializerMatrix<ElementType> tmp( list, columns() );
10254 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10255 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10256 }
10257 }
10258
10259 decltype(auto) left( derestrict( *this ) );
10260 size_t i( 0UL );
10261
10262 for( const auto& rowList : list ) {
10263 size_t j( 0UL );
10264 for( const auto& element : rowList ) {
10265 left(i,j) = element;
10266 ++j;
10267 }
10268 for( ; j<columns(); ++j ) {
10269 reset( left(i,j) );
10270 }
10271 ++i;
10272 }
10273
10274 return *this;
10275}
10277//*************************************************************************************************
10278
10279
10280//*************************************************************************************************
10295template< typename MT // Type of the dense matrix
10296 , size_t... CSAs > // Compile time submatrix arguments
10297inline Submatrix<MT,aligned,true,true,CSAs...>&
10298 Submatrix<MT,aligned,true,true,CSAs...>::operator=( const Submatrix& rhs )
10299{
10302
10303 if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
10304 return *this;
10305
10306 if( rows() != rhs.rows() || columns() != rhs.columns() ) {
10307 BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
10308 }
10309
10310 if( !tryAssign( matrix_, rhs, row(), column() ) ) {
10311 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10312 }
10313
10314 decltype(auto) left( derestrict( *this ) );
10315
10316 if( rhs.canAlias( this ) ) {
10317 const ResultType tmp( rhs );
10318 smpAssign( left, tmp );
10319 }
10320 else {
10321 smpAssign( left, rhs );
10322 }
10323
10324 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10325
10326 return *this;
10327}
10329//*************************************************************************************************
10330
10331
10332//*************************************************************************************************
10346template< typename MT // Type of the dense matrix
10347 , size_t... CSAs > // Compile time submatrix arguments
10348template< typename MT2 // Type of the right-hand side matrix
10349 , bool SO > // Storage order of the right-hand side matrix
10350inline Submatrix<MT,aligned,true,true,CSAs...>&
10351 Submatrix<MT,aligned,true,true,CSAs...>::operator=( const Matrix<MT2,SO>& rhs )
10352{
10354
10355 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
10356 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10357 }
10358
10359 using Right = If_t< IsRestricted_v<MT>, CompositeType_t<MT2>, const MT2& >;
10360 Right right( *rhs );
10361
10362 if( !tryAssign( matrix_, right, row(), column() ) ) {
10363 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10364 }
10365
10366 decltype(auto) left( derestrict( *this ) );
10367
10368 if( IsReference_v<Right> && right.canAlias( this ) ) {
10369 const ResultType_t<MT2> tmp( right );
10370 if( IsSparseMatrix_v<MT2> )
10371 reset();
10372 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
10373 }
10374 else {
10375 if( IsSparseMatrix_v<MT2> )
10376 reset();
10377 smpAssign( left, transIf< IsSymmetric_v<This> >( right ) );
10378 }
10379
10380 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10381
10382 return *this;
10383}
10385//*************************************************************************************************
10386
10387
10388//*************************************************************************************************
10402template< typename MT // Type of the dense matrix
10403 , size_t... CSAs > // Compile time submatrix arguments
10404template< typename MT2 // Type of the right-hand side matrix
10405 , bool SO > // Storage order of the right-hand side matrix
10406inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
10407 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10408{
10412
10413 using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
10414
10417
10418 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
10419 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10420 }
10421
10422 if( !tryAddAssign( matrix_, *rhs, row(), column() ) ) {
10423 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10424 }
10425
10426 decltype(auto) left( derestrict( *this ) );
10427
10428 if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) || (*rhs).canAlias( this ) ) {
10429 const AddType tmp( *this + (*rhs) );
10430 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
10431 }
10432 else {
10433 smpAddAssign( left, transIf< IsSymmetric_v<This> >( *rhs ) );
10434 }
10435
10436 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10437
10438 return *this;
10439}
10441//*************************************************************************************************
10442
10443
10444//*************************************************************************************************
10458template< typename MT // Type of the dense matrix
10459 , size_t... CSAs > // Compile time submatrix arguments
10460template< typename MT2 // Type of the right-hand side matrix
10461 , bool SO > // Storage order of the right-hand side matrix
10462inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
10463 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10464{
10468
10469 using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
10470
10473
10474 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
10475 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10476 }
10477
10478 const AddType tmp( *this + (*rhs) );
10479
10480 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10481 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10482 }
10483
10484 decltype(auto) left( derestrict( *this ) );
10485
10486 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
10487
10488 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10489
10490 return *this;
10491}
10493//*************************************************************************************************
10494
10495
10496//*************************************************************************************************
10510template< typename MT // Type of the dense matrix
10511 , size_t... CSAs > // Compile time submatrix arguments
10512template< typename MT2 // Type of the right-hand side matrix
10513 , bool SO > // Storage order of the right-hand side matrix
10514inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
10515 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10516{
10520
10521 using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
10522
10525
10526 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
10527 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10528 }
10529
10530 if( !trySubAssign( matrix_, *rhs, row(), column() ) ) {
10531 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10532 }
10533
10534 decltype(auto) left( derestrict( *this ) );
10535
10536 if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) || (*rhs).canAlias( this ) ) {
10537 const SubType tmp( *this - (*rhs ) );
10538 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
10539 }
10540 else {
10541 smpSubAssign( left, transIf< IsSymmetric_v<This> >( *rhs ) );
10542 }
10543
10544 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10545
10546 return *this;
10547}
10549//*************************************************************************************************
10550
10551
10552//*************************************************************************************************
10566template< typename MT // Type of the dense matrix
10567 , size_t... CSAs > // Compile time submatrix arguments
10568template< typename MT2 // Type of the right-hand side matrix
10569 , bool SO > // Storage order of the right-hand side matrix
10570inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
10571 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10572{
10576
10577 using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
10578
10581
10582 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
10583 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10584 }
10585
10586 const SubType tmp( *this - (*rhs) );
10587
10588 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10589 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10590 }
10591
10592 decltype(auto) left( derestrict( *this ) );
10593
10594 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
10595
10596 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10597
10598 return *this;
10599}
10601//*************************************************************************************************
10602
10603
10604//*************************************************************************************************
10618template< typename MT // Type of the dense matrix
10619 , size_t... CSAs > // Compile time submatrix arguments
10620template< typename MT2 // Type of the right-hand side matrix
10621 , bool SO > // Storage order of the right-hand side matrix
10622inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
10623 -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10624{
10628
10629 using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
10630
10632
10633 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
10634 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10635 }
10636
10637 if( !trySchurAssign( matrix_, *rhs, row(), column() ) ) {
10638 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10639 }
10640
10641 decltype(auto) left( derestrict( *this ) );
10642
10643 if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) || (*rhs).canAlias( this ) ) {
10644 const SchurType tmp( *this % (*rhs) );
10645 if( IsSparseMatrix_v<SchurType> )
10646 reset();
10647 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
10648 }
10649 else {
10650 smpSchurAssign( left, transIf< IsSymmetric_v<This> >( *rhs ) );
10651 }
10652
10653 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10654
10655 return *this;
10656}
10658//*************************************************************************************************
10659
10660
10661//*************************************************************************************************
10675template< typename MT // Type of the dense matrix
10676 , size_t... CSAs > // Compile time submatrix arguments
10677template< typename MT2 // Type of the right-hand side matrix
10678 , bool SO > // Storage order of the right-hand side matrix
10679inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
10680 -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10681{
10685
10686 using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
10687
10689
10690 if( rows() != (*rhs).rows() || columns() != (*rhs).columns() ) {
10691 BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10692 }
10693
10694 const SchurType tmp( *this % (*rhs) );
10695
10696 if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10697 BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10698 }
10699
10700 decltype(auto) left( derestrict( *this ) );
10701
10702 if( IsSparseMatrix_v<SchurType> ) {
10703 reset();
10704 }
10705
10706 smpAssign( left, transIf< IsSymmetric_v<This> >( tmp ) );
10707
10708 BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10709
10710 return *this;
10711}
10713//*************************************************************************************************
10714
10715
10716
10717
10718//=================================================================================================
10719//
10720// UTILITY FUNCTIONS
10721//
10722//=================================================================================================
10723
10724//*************************************************************************************************
10730template< typename MT // Type of the dense matrix
10731 , size_t... CSAs > // Compile time submatrix arguments
10732inline MT& Submatrix<MT,aligned,true,true,CSAs...>::operand() noexcept
10733{
10734 return matrix_;
10735}
10737//*************************************************************************************************
10738
10739
10740//*************************************************************************************************
10746template< typename MT // Type of the dense matrix
10747 , size_t... CSAs > // Compile time submatrix arguments
10748inline const MT& Submatrix<MT,aligned,true,true,CSAs...>::operand() const noexcept
10749{
10750 return matrix_;
10751}
10753//*************************************************************************************************
10754
10755
10756//*************************************************************************************************
10765template< typename MT // Type of the dense matrix
10766 , size_t... CSAs > // Compile time submatrix arguments
10767inline size_t Submatrix<MT,aligned,true,true,CSAs...>::spacing() const noexcept
10768{
10769 return matrix_.spacing();
10770}
10772//*************************************************************************************************
10773
10774
10775//*************************************************************************************************
10781template< typename MT // Type of the dense matrix
10782 , size_t... CSAs > // Compile time submatrix arguments
10783inline size_t Submatrix<MT,aligned,true,true,CSAs...>::capacity() const noexcept
10784{
10785 return rows() * columns();
10786}
10788//*************************************************************************************************
10789
10790
10791//*************************************************************************************************
10798template< typename MT // Type of the dense matrix
10799 , size_t... CSAs > // Compile time submatrix arguments
10800inline size_t Submatrix<MT,aligned,true,true,CSAs...>::capacity( size_t j ) const noexcept
10801{
10802 MAYBE_UNUSED( j );
10803
10804 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
10805
10806 return rows();
10807}
10809//*************************************************************************************************
10810
10811
10812//*************************************************************************************************
10818template< typename MT // Type of the dense matrix
10819 , size_t... CSAs > // Compile time submatrix arguments
10821{
10822 const size_t iend( row() + rows() );
10823 const size_t jend( column() + columns() );
10824 size_t nonzeros( 0UL );
10825
10826 for( size_t j=column(); j<jend; ++j )
10827 for( size_t i=row(); i<iend; ++i )
10828 if( !isDefault( matrix_(i,j) ) )
10829 ++nonzeros;
10830
10831 return nonzeros;
10832}
10834//*************************************************************************************************
10835
10836
10837//*************************************************************************************************
10844template< typename MT // Type of the dense matrix
10845 , size_t... CSAs > // Compile time submatrix arguments
10846inline size_t Submatrix<MT,aligned,true,true,CSAs...>::nonZeros( size_t j ) const
10847{
10848 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
10849
10850 const size_t iend( row() + rows() );
10851 size_t nonzeros( 0UL );
10852
10853 for( size_t i=row(); i<iend; ++i )
10854 if( !isDefault( matrix_(i,column()+j) ) )
10855 ++nonzeros;
10856
10857 return nonzeros;
10858}
10860//*************************************************************************************************
10861
10862
10863//*************************************************************************************************
10869template< typename MT // Type of the dense matrix
10870 , size_t... CSAs > // Compile time submatrix arguments
10872{
10873 using blaze::clear;
10874
10875 for( size_t j=column(); j<column()+columns(); ++j )
10876 {
10877 const size_t ibegin( ( IsLower_v<MT> )
10878 ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
10879 ?( max( j+1UL, row() ) )
10880 :( max( j, row() ) ) )
10881 :( row() ) );
10882 const size_t iend ( ( IsUpper_v<MT> )
10883 ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
10884 ?( min( j, row()+rows() ) )
10885 :( min( j+1UL, row()+rows() ) ) )
10886 :( row()+rows() ) );
10887
10888 for( size_t i=ibegin; i<iend; ++i )
10889 clear( matrix_(i,j) );
10890 }
10891}
10893//*************************************************************************************************
10894
10895
10896//*************************************************************************************************
10903template< typename MT // Type of the dense matrix
10904 , size_t... CSAs > // Compile time submatrix arguments
10906{
10907 using blaze::clear;
10908
10909 BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
10910
10911 const size_t ibegin( ( IsLower_v<MT> )
10912 ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
10913 ?( max( j+1UL, row() ) )
10914 :( max( j, row() ) ) )
10915 :( row() ) );
10916 const size_t iend ( ( IsUpper_v<MT> )
10917 ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
10918 ?( min( j, row()+rows() ) )
10919 :( min( j+1UL, row()+rows() ) ) )
10920 :( row()+rows() ) );
10921
10922 for( size_t i=ibegin; i<iend; ++i )
10923 clear( matrix_(i,column()+j) );
10924}
10926//*************************************************************************************************
10927
10928
10929//*************************************************************************************************
10939template< typename MT // Type of the dense matrix
10940 , size_t... CSAs > // Compile time submatrix arguments
10941inline bool Submatrix<MT,aligned,true,true,CSAs...>::hasOverlap() const noexcept
10942{
10943 BLAZE_INTERNAL_ASSERT( IsSymmetric_v<MT> || IsHermitian_v<MT>, "Invalid matrix detected" );
10944
10945 if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
10946 return false;
10947 else return true;
10948}
10950//*************************************************************************************************
10951
10952
10953
10954
10955//=================================================================================================
10956//
10957// NUMERIC FUNCTIONS
10958//
10959//=================================================================================================
10960
10961//*************************************************************************************************
10979template< typename MT // Type of the dense matrix
10980 , size_t... CSAs > // Compile time submatrix arguments
10981inline Submatrix<MT,aligned,true,true,CSAs...>&
10983{
10984 if( rows() != columns() ) {
10985 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
10986 }
10987
10988 if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
10989 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
10990 }
10991
10992 decltype(auto) left( derestrict( *this ) );
10993 const ResultType tmp( trans( *this ) );
10994
10995 smpAssign( left, tmp );
10996
10997 return *this;
10998}
11000//*************************************************************************************************
11001
11002
11003//*************************************************************************************************
11021template< typename MT // Type of the dense matrix
11022 , size_t... CSAs > // Compile time submatrix arguments
11023inline Submatrix<MT,aligned,true,true,CSAs...>&
11025{
11026 if( rows() != columns() ) {
11027 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
11028 }
11029
11030 if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
11031 BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
11032 }
11033
11034 decltype(auto) left( derestrict( *this ) );
11035 const ResultType tmp( ctrans( *this ) );
11036
11037 smpAssign( left, tmp );
11038
11039 return *this;
11040}
11042//*************************************************************************************************
11043
11044
11045//*************************************************************************************************
11058template< typename MT // Type of the dense matrix
11059 , size_t... CSAs > // Compile time submatrix arguments
11060template< typename Other > // Data type of the scalar value
11061inline Submatrix<MT,aligned,true,true,CSAs...>&
11062 Submatrix<MT,aligned,true,true,CSAs...>::scale( const Other& scalar )
11063{
11065
11066 const size_t jend( column() + columns() );
11067
11068 for( size_t j=column(); j<jend; ++j )
11069 {
11070 const size_t ibegin( ( IsLower_v<MT> )
11071 ?( ( IsStrictlyLower_v<MT> )
11072 ?( max( j+1UL, row() ) )
11073 :( max( j, row() ) ) )
11074 :( row() ) );
11075 const size_t iend ( ( IsUpper_v<MT> )
11076 ?( ( IsStrictlyUpper_v<MT> )
11077 ?( min( j, row()+rows() ) )
11078 :( min( j+1UL, row()+rows() ) ) )
11079 :( row()+rows() ) );
11080
11081 for( size_t i=ibegin; i<iend; ++i )
11082 matrix_(i,j) *= scalar;
11083 }
11084
11085 return *this;
11086}
11088//*************************************************************************************************
11089
11090
11091
11092
11093//=================================================================================================
11094//
11095// EXPRESSION TEMPLATE EVALUATION FUNCTIONS
11096//
11097//=================================================================================================
11098
11099//*************************************************************************************************
11110template< typename MT // Type of the dense matrix
11111 , size_t... CSAs > // Compile time submatrix arguments
11112template< typename Other > // Data type of the foreign expression
11113inline bool Submatrix<MT,aligned,true,true,CSAs...>::canAlias( const Other* alias ) const noexcept
11114{
11115 return matrix_.isAliased( &unview( *alias ) );
11116}
11118//*************************************************************************************************
11119
11120
11121//*************************************************************************************************
11132template< typename MT // Type of the dense matrix
11133 , size_t... CSAs > // Compile time submatrix arguments
11134template< typename MT2 // Data type of the foreign dense submatrix
11135 , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
11136 , bool SO2 // Storage order of the foreign dense submatrix
11137 , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
11138inline bool
11139 Submatrix<MT,aligned,true,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
11140{
11141 return ( matrix_.isAliased( &alias->matrix_ ) &&
11142 ( row() + rows() > alias->row() ) &&
11143 ( row() < alias->row() + alias->rows() ) &&
11144 ( column() + columns() > alias->column() ) &&
11145 ( column() < alias->column() + alias->columns() ) );
11146}
11148//*************************************************************************************************
11149
11150
11151//*************************************************************************************************
11162template< typename MT // Type of the dense matrix
11163 , size_t... CSAs > // Compile time submatrix arguments
11164template< typename Other > // Data type of the foreign expression
11165inline bool Submatrix<MT,aligned,true,true,CSAs...>::isAliased( const Other* alias ) const noexcept
11166{
11167 return matrix_.isAliased( &unview( *alias ) );
11168}
11170//*************************************************************************************************
11171
11172
11173//*************************************************************************************************
11184template< typename MT // Type of the dense matrix
11185 , size_t... CSAs > // Compile time submatrix arguments
11186template< typename MT2 // Data type of the foreign dense submatrix
11187 , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
11188 , bool SO2 // Storage order of the foreign dense submatrix
11189 , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
11190inline bool
11191 Submatrix<MT,aligned,true,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
11192{
11193 return ( matrix_.isAliased( &alias->matrix_ ) &&
11194 ( row() + rows() > alias->row() ) &&
11195 ( row() < alias->row() + alias->rows() ) &&
11196 ( column() + columns() > alias->column() ) &&
11197 ( column() < alias->column() + alias->columns() ) );
11198}
11200//*************************************************************************************************
11201
11202
11203//*************************************************************************************************
11213template< typename MT // Type of the dense matrix
11214 , size_t... CSAs > // Compile time submatrix arguments
11215inline bool Submatrix<MT,aligned,true,true,CSAs...>::isAligned() const noexcept
11216{
11217 return true;
11218}
11220//*************************************************************************************************
11221
11222
11223//*************************************************************************************************
11234template< typename MT // Type of the dense matrix
11235 , size_t... CSAs > // Compile time submatrix arguments
11236inline bool Submatrix<MT,aligned,true,true,CSAs...>::canSMPAssign() const noexcept
11237{
11238 return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
11239}
11241//*************************************************************************************************
11242
11243
11244//*************************************************************************************************
11259template< typename MT // Type of the dense matrix
11260 , size_t... CSAs > // Compile time submatrix arguments
11261BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,true,true,CSAs...>::SIMDType
11262 Submatrix<MT,aligned,true,true,CSAs...>::load( size_t i, size_t j ) const noexcept
11263{
11264 return loada( i, j );
11265}
11267//*************************************************************************************************
11268
11269
11270//*************************************************************************************************
11285template< typename MT // Type of the dense matrix
11286 , size_t... CSAs > // Compile time submatrix arguments
11287BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,true,true,CSAs...>::SIMDType
11288 Submatrix<MT,aligned,true,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
11289{
11291
11292 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11293 BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11294 BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11295 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11296
11297 return matrix_.loada( row()+i, column()+j );
11298}
11300//*************************************************************************************************
11301
11302
11303//*************************************************************************************************
11318template< typename MT // Type of the dense matrix
11319 , size_t... CSAs > // Compile time submatrix arguments
11320BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,true,true,CSAs...>::SIMDType
11321 Submatrix<MT,aligned,true,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
11322{
11324
11325 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11326 BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11327 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11328
11329 return matrix_.loadu( row()+i, column()+j );
11330}
11332//*************************************************************************************************
11333
11334
11335//*************************************************************************************************
11351template< typename MT // Type of the dense matrix
11352 , size_t... CSAs > // Compile time submatrix arguments
11354 Submatrix<MT,aligned,true,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
11355{
11356 storea( i, j, value );
11357}
11359//*************************************************************************************************
11360
11361
11362//*************************************************************************************************
11378template< typename MT // Type of the dense matrix
11379 , size_t... CSAs > // Compile time submatrix arguments
11381 Submatrix<MT,aligned,true,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
11382{
11384
11385 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11386 BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11387 BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11388 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11389
11390 matrix_.storea( row()+i, column()+j, value );
11391}
11393//*************************************************************************************************
11394
11395
11396//*************************************************************************************************
11412template< typename MT // Type of the dense matrix
11413 , size_t... CSAs > // Compile time submatrix arguments
11415 Submatrix<MT,aligned,true,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
11416{
11418
11419 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11420 BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11421 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11422
11423 matrix_.storeu( row()+i, column()+j, value );
11424}
11426//*************************************************************************************************
11427
11428
11429//*************************************************************************************************
11446template< typename MT // Type of the dense matrix
11447 , size_t... CSAs > // Compile time submatrix arguments
11449 Submatrix<MT,aligned,true,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
11450{
11452
11453 BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11454 BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11455 BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11456 BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11457
11458 matrix_.stream( row()+i, column()+j, value );
11459}
11461//*************************************************************************************************
11462
11463
11464//*************************************************************************************************
11476template< typename MT // Type of the dense matrix
11477 , size_t... CSAs > // Compile time submatrix arguments
11478template< typename MT2 > // Type of the right-hand side dense matrix
11479inline auto Submatrix<MT,aligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
11480 -> DisableIf_t< VectorizedAssign_v<MT2> >
11481{
11482 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
11483 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
11484
11485 const size_t ipos( prevMultiple( rows(), 2UL ) );
11486 BLAZE_INTERNAL_ASSERT( ipos <= rows(), "Invalid end calculation" );
11487
11488 for( size_t j=0UL; j<columns(); ++j ) {
11489 for( size_t i=0UL; i<ipos; i+=2UL ) {
11490 matrix_(row()+i ,column()+j) = (*rhs)(i ,j);
11491 matrix_(row()+i+1UL,column()+j) = (*rhs)(i+1UL,j);
11492 }
11493 if( ipos < rows() ) {
11494 matrix_(row()+ipos,column()+j) = (*rhs)(ipos,j);
11495 }
11496 }
11497}
11499//*************************************************************************************************
11500
11501
11502//*************************************************************************************************
11514template< typename MT // Type of the dense matrix
11515 , size_t... CSAs > // Compile time submatrix arguments
11516template< typename MT2 > // Type of the right-hand side dense matrix
11517inline auto Submatrix<MT,aligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
11518 -> EnableIf_t< VectorizedAssign_v<MT2> >
11519{
11521
11522 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
11523 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
11524
11525 const size_t ipos( prevMultiple( rows(), SIMDSIZE ) );
11526 BLAZE_INTERNAL_ASSERT( ipos <= rows(), "Invalid end calculation" );
11527
11528 if( useStreaming &&
11529 rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
11530 !(*rhs).isAliased( this ) )
11531 {
11532 for( size_t j=0UL; j<columns(); ++j )
11533 {
11534 size_t i( 0UL );
11535 Iterator left( begin(j) );
11536 ConstIterator_t<MT2> right( (*rhs).begin(j) );
11537
11538 for( ; i<ipos; i+=SIMDSIZE ) {
11539 left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11540 }
11541 for( ; i<rows(); ++i ) {
11542 *left = *right; ++left; ++right;
11543 }
11544 }
11545 }
11546 else
11547 {
11548 for( size_t j=0UL; j<columns(); ++j )
11549 {
11550 size_t i( 0UL );
11551 Iterator left( begin(j) );
11552 ConstIterator_t<MT2> right( (*rhs).begin(j) );
11553
11554 for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
11555 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11556 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11557 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11558 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11559 }
11560 for( ; i<ipos; i+=SIMDSIZE ) {
11561 left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11562 }
11563 for( ; i<rows(); ++i ) {
11564 *left = *right; ++left; ++right;
11565 }
11566 }
11567 }
11568}
11570//*************************************************************************************************
11571
11572
11573//*************************************************************************************************
11585template< typename MT // Type of the dense matrix
11586 , size_t... CSAs > // Compile time submatrix arguments
11587template< typename MT2 > // Type of the right-hand side dense matrix
11588inline void Submatrix<MT,aligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
11589{
11591
11592 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
11593 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
11594
11595 constexpr size_t block( BLOCK_SIZE );
11596
11597 for( size_t jj=0UL; jj<columns(); jj+=block ) {
11598 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
11599 for( size_t ii=0UL; ii<rows(); ii+=block ) {
11600 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
11601 for( size_t j=jj; j<jend; ++j ) {
11602 for( size_t i=ii; i<iend; ++i ) {
11603 matrix_(row()+i,column()+j) = (*rhs)(i,j);
11604 }
11605 }
11606 }
11607 }
11608}
11610//*************************************************************************************************
11611
11612
11613//*************************************************************************************************
11625template< typename MT // Type of the dense matrix
11626 , size_t... CSAs > // Compile time submatrix arguments
11627template< typename MT2 > // Type of the right-hand side sparse matrix
11628inline void Submatrix<MT,aligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
11629{
11630 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
11631 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
11632
11633 for( size_t j=0UL; j<columns(); ++j )
11634 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element )
11635 matrix_(row()+element->index(),column()+j) = element->value();
11636}
11638//*************************************************************************************************
11639
11640
11641//*************************************************************************************************
11653template< typename MT // Type of the dense matrix
11654 , size_t... CSAs > // Compile time submatrix arguments
11655template< typename MT2 > // Type of the right-hand side sparse matrix
11656inline void Submatrix<MT,aligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
11657{
11659
11660 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
11661 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
11662
11663 for( size_t i=0UL; i<rows(); ++i )
11664 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element )
11665 matrix_(row()+i,column()+element->index()) = element->value();
11666}
11668//*************************************************************************************************
11669
11670
11671//*************************************************************************************************
11683template< typename MT // Type of the dense matrix
11684 , size_t... CSAs > // Compile time submatrix arguments
11685template< typename MT2 > // Type of the right-hand side dense matrix
11686inline auto Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
11687 -> DisableIf_t< VectorizedAddAssign_v<MT2> >
11688{
11689 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
11690 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
11691
11692 const size_t ipos( prevMultiple( rows(), 2UL ) );
11693 BLAZE_INTERNAL_ASSERT( ipos <= rows(), "Invalid end calculation" );
11694
11695 for( size_t j=0UL; j<columns(); ++j )
11696 {
11697 if( IsDiagonal_v<MT2> ) {
11698 matrix_(row()+j,column()+j) += (*rhs)(j,j);
11699 }
11700 else {
11701 for( size_t i=0UL; i<ipos; i+=2UL ) {
11702 matrix_(row()+i ,column()+j) += (*rhs)(i ,j);
11703 matrix_(row()+i+1UL,column()+j) += (*rhs)(i+1UL,j);
11704 }
11705 if( ipos < rows() ) {
11706 matrix_(row()+ipos,column()+j) += (*rhs)(ipos,j);
11707 }
11708 }
11709 }
11710}
11712//*************************************************************************************************
11713
11714
11715//*************************************************************************************************
11727template< typename MT // Type of the dense matrix
11728 , size_t... CSAs > // Compile time submatrix arguments
11729template< typename MT2 > // Type of the right-hand side dense matrix
11730inline auto Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
11731 -> EnableIf_t< VectorizedAddAssign_v<MT2> >
11732{
11734
11735 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
11736 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
11737
11738 for( size_t j=0UL; j<columns(); ++j )
11739 {
11740 const size_t ibegin( ( IsLower_v<MT> )
11741 ?( prevMultiple( ( IsStrictlyLower_v<MT> ? j+1UL : j ), SIMDSIZE ) )
11742 :( 0UL ) );
11743 const size_t iend ( ( IsUpper_v<MT> )
11744 ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
11745 :( rows() ) );
11746 BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
11747
11748 const size_t ipos( prevMultiple( iend, SIMDSIZE ) );
11749 BLAZE_INTERNAL_ASSERT( ipos <= iend, "Invalid end calculation" );
11750
11751 size_t i( ibegin );
11752 Iterator left( begin(j) + ibegin );
11753 ConstIterator_t<MT2> right( (*rhs).begin(j) + ibegin );
11754
11755 for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
11756 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11757 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11758 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11759 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11760 }
11761 for( ; i<ipos; i+=SIMDSIZE ) {
11762 left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11763 }
11764 for( ; i<iend; ++i ) {
11765 *left += *right; ++left; ++right;
11766 }
11767 }
11768}
11770//*************************************************************************************************
11771
11772
11773//*************************************************************************************************
11785template< typename MT // Type of the dense matrix
11786 , size_t... CSAs > // Compile time submatrix arguments
11787template< typename MT2 > // Type of the right-hand side dense matrix
11788inline void Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
11789{
11791
11792 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
11793 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
11794
11795 constexpr size_t block( BLOCK_SIZE );
11796
11797 for( size_t jj=0UL; jj<columns(); jj+=block ) {
11798 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
11799 for( size_t ii=0UL; ii<rows(); ii+=block ) {
11800 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
11801 for( size_t j=jj; j<jend; ++j ) {
11802 for( size_t i=ii; i<iend; ++i ) {
11803 matrix_(row()+i,column()+j) += (*rhs)(i,j);
11804 }
11805 }
11806 }
11807 }
11808}
11810//*************************************************************************************************
11811
11812
11813//*************************************************************************************************
11825template< typename MT // Type of the dense matrix
11826 , size_t... CSAs > // Compile time submatrix arguments
11827template< typename MT2 > // Type of the right-hand side sparse matrix
11828inline void Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
11829{
11830 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
11831 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
11832
11833 for( size_t j=0UL; j<columns(); ++j )
11834 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element )
11835 matrix_(row()+element->index(),column()+j) += element->value();
11836}
11838//*************************************************************************************************
11839
11840
11841//*************************************************************************************************
11853template< typename MT // Type of the dense matrix
11854 , size_t... CSAs > // Compile time submatrix arguments
11855template< typename MT2 > // Type of the right-hand side sparse matrix
11856inline void Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
11857{
11859
11860 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
11861 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
11862
11863 for( size_t i=0UL; i<rows(); ++i )
11864 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element )
11865 matrix_(row()+i,column()+element->index()) += element->value();
11866}
11868//*************************************************************************************************
11869
11870
11871//*************************************************************************************************
11883template< typename MT // Type of the dense matrix
11884 , size_t... CSAs > // Compile time submatrix arguments
11885template< typename MT2 > // Type of the right-hand side dense matrix
11886inline auto Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
11887 -> DisableIf_t< VectorizedSubAssign_v<MT2> >
11888{
11889 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
11890 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
11891
11892 const size_t ipos( prevMultiple( rows(), 2UL ) );
11893 BLAZE_INTERNAL_ASSERT( ipos <= rows(), "Invalid end calculation" );
11894
11895 for( size_t j=0UL; j<columns(); ++j )
11896 {
11897 if( IsDiagonal_v<MT2> ) {
11898 matrix_(row()+j,column()+j) -= (*rhs)(j,j);
11899 }
11900 else {
11901 for( size_t i=0UL; i<ipos; i+=2UL ) {
11902 matrix_(row()+i ,column()+j) -= (*rhs)(i ,j);
11903 matrix_(row()+i+1UL,column()+j) -= (*rhs)(i+1UL,j);
11904 }
11905 if( ipos < rows() ) {
11906 matrix_(row()+ipos,column()+j) -= (*rhs)(ipos,j);
11907 }
11908 }
11909 }
11910}
11912//*************************************************************************************************
11913
11914
11915//*************************************************************************************************
11927template< typename MT // Type of the dense matrix
11928 , size_t... CSAs > // Compile time submatrix arguments
11929template< typename MT2 > // Type of the right-hand side dense matrix
11930inline auto Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
11931 -> EnableIf_t< VectorizedSubAssign_v<MT2> >
11932{
11934
11935 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
11936 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
11937
11938 for( size_t j=0UL; j<columns(); ++j )
11939 {
11940 const size_t ibegin( ( IsLower_v<MT> )
11941 ?( prevMultiple( ( IsStrictlyLower_v<MT> ? j+1UL : j ), SIMDSIZE ) )
11942 :( 0UL ) );
11943 const size_t iend ( ( IsUpper_v<MT> )
11944 ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
11945 :( rows() ) );
11946 BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
11947
11948 const size_t ipos( prevMultiple( iend, SIMDSIZE ) );
11949 BLAZE_INTERNAL_ASSERT( ipos <= iend, "Invalid end calculation" );
11950
11951 size_t i( ibegin );
11952 Iterator left( begin(j) + ibegin );
11953 ConstIterator_t<MT2> right( (*rhs).begin(j) + ibegin );
11954
11955 for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
11956 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11957 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11958 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11959 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11960 }
11961 for( ; i<ipos; i+=SIMDSIZE ) {
11962 left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11963 }
11964 for( ; i<iend; ++i ) {
11965 *left -= *right; ++left; ++right;
11966 }
11967 }
11968}
11970//*************************************************************************************************
11971
11972
11973//*************************************************************************************************
11985template< typename MT // Type of the dense matrix
11986 , size_t... CSAs > // Compile time submatrix arguments
11987template< typename MT2 > // Type of the right-hand side dense matrix
11988inline void Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
11989{
11991
11992 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
11993 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
11994
11995 constexpr size_t block( BLOCK_SIZE );
11996
11997 for( size_t jj=0UL; jj<columns(); jj+=block ) {
11998 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
11999 for( size_t ii=0UL; ii<rows(); ii+=block ) {
12000 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
12001 for( size_t j=jj; j<jend; ++j ) {
12002 for( size_t i=ii; i<iend; ++i ) {
12003 matrix_(row()+i,column()+j) -= (*rhs)(i,j);
12004 }
12005 }
12006 }
12007 }
12008}
12010//*************************************************************************************************
12011
12012
12013//*************************************************************************************************
12025template< typename MT // Type of the dense matrix
12026 , size_t... CSAs > // Compile time submatrix arguments
12027template< typename MT2 > // Type of the right-hand side sparse matrix
12028inline void Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
12029{
12030 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
12031 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
12032
12033 for( size_t j=0UL; j<columns(); ++j )
12034 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element )
12035 matrix_(row()+element->index(),column()+j) -= element->value();
12036}
12038//*************************************************************************************************
12039
12040
12041//*************************************************************************************************
12053template< typename MT // Type of the dense matrix
12054 , size_t... CSAs > // Compile time submatrix arguments
12055template< typename MT2 > // Type of the right-hand side sparse matrix
12056inline void Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
12057{
12059
12060 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
12061 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
12062
12063 for( size_t i=0UL; i<rows(); ++i )
12064 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element )
12065 matrix_(row()+i,column()+element->index()) -= element->value();
12066}
12068//*************************************************************************************************
12069
12070
12071//*************************************************************************************************
12083template< typename MT // Type of the dense matrix
12084 , size_t... CSAs > // Compile time submatrix arguments
12085template< typename MT2 > // Type of the right-hand side dense matrix
12086inline auto Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
12087 -> DisableIf_t< VectorizedSchurAssign_v<MT2> >
12088{
12089 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
12090 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
12091
12092 const size_t ipos( prevMultiple( rows(), 2UL ) );
12093 BLAZE_INTERNAL_ASSERT( ipos <= rows(), "Invalid end calculation" );
12094
12095 for( size_t j=0UL; j<columns(); ++j ) {
12096 for( size_t i=0UL; i<ipos; i+=2UL ) {
12097 matrix_(row()+i ,column()+j) *= (*rhs)(i ,j);
12098 matrix_(row()+i+1UL,column()+j) *= (*rhs)(i+1UL,j);
12099 }
12100 if( ipos < rows() ) {
12101 matrix_(row()+ipos,column()+j) *= (*rhs)(ipos,j);
12102 }
12103 }
12104}
12106//*************************************************************************************************
12107
12108
12109//*************************************************************************************************
12122template< typename MT // Type of the dense matrix
12123 , size_t... CSAs > // Compile time submatrix arguments
12124template< typename MT2 > // Type of the right-hand side dense matrix
12125inline auto Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
12126 -> EnableIf_t< VectorizedSchurAssign_v<MT2> >
12127{
12129
12130 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
12131 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
12132
12133 for( size_t j=0UL; j<columns(); ++j )
12134 {
12135 const size_t ipos( prevMultiple( rows(), SIMDSIZE ) );
12136 BLAZE_INTERNAL_ASSERT( ipos <= rows(), "Invalid end calculation" );
12137
12138 size_t i( 0UL );
12139 Iterator left( begin(j) );
12140 ConstIterator_t<MT2> right( (*rhs).begin(j) );
12141
12142 for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
12143 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12144 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12145 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12146 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12147 }
12148 for( ; i<ipos; i+=SIMDSIZE ) {
12149 left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12150 }
12151 for( ; i<rows(); ++i ) {
12152 *left *= *right; ++left; ++right;
12153 }
12154 }
12155}
12157//*************************************************************************************************
12158
12159
12160//*************************************************************************************************
12172template< typename MT // Type of the dense matrix
12173 , size_t... CSAs > // Compile time submatrix arguments
12174template< typename MT2 > // Type of the right-hand side dense matrix
12175inline void Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
12176{
12178
12179 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
12180 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
12181
12182 constexpr size_t block( BLOCK_SIZE );
12183
12184 for( size_t jj=0UL; jj<columns(); jj+=block ) {
12185 const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
12186 for( size_t ii=0UL; ii<rows(); ii+=block ) {
12187 const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
12188 for( size_t j=jj; j<jend; ++j ) {
12189 for( size_t i=ii; i<iend; ++i ) {
12190 matrix_(row()+i,column()+j) *= (*rhs)(i,j);
12191 }
12192 }
12193 }
12194 }
12195}
12197//*************************************************************************************************
12198
12199
12200//*************************************************************************************************
12212template< typename MT // Type of the dense matrix
12213 , size_t... CSAs > // Compile time submatrix arguments
12214template< typename MT2 > // Type of the right-hand side sparse matrix
12215inline void Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
12216{
12217 using blaze::reset;
12218
12219 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
12220 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
12221
12222 for( size_t j=0UL; j<columns(); ++j )
12223 {
12224 size_t i( 0UL );
12225
12226 for( ConstIterator_t<MT2> element=(*rhs).begin(j); element!=(*rhs).end(j); ++element ) {
12227 for( ; i<element->index(); ++i )
12228 reset( matrix_(row()+i,column()+j) );
12229 matrix_(row()+i,column()+j) *= element->value();
12230 ++i;
12231 }
12232
12233 for( ; i<rows(); ++i ) {
12234 reset( matrix_(row()+i,column()+j) );
12235 }
12236 }
12237}
12239//*************************************************************************************************
12240
12241
12242//*************************************************************************************************
12254template< typename MT // Type of the dense matrix
12255 , size_t... CSAs > // Compile time submatrix arguments
12256template< typename MT2 > // Type of the right-hand side sparse matrix
12257inline void Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
12258{
12259 using blaze::reset;
12260
12262
12263 BLAZE_INTERNAL_ASSERT( rows() == (*rhs).rows() , "Invalid number of rows" );
12264 BLAZE_INTERNAL_ASSERT( columns() == (*rhs).columns(), "Invalid number of columns" );
12265
12266 for( size_t i=0UL; i<rows(); ++i )
12267 {
12268 size_t j( 0UL );
12269
12270 for( ConstIterator_t<MT2> element=(*rhs).begin(i); element!=(*rhs).end(i); ++element ) {
12271 for( ; j<element->index(); ++j )
12272 reset( matrix_(row()+i,column()+j) );
12273 matrix_(row()+i,column()+j) *= element->value();
12274 ++j;
12275 }
12276
12277 for( ; j<columns(); ++j ) {
12278 reset( matrix_(row()+i,column()+j) );
12279 }
12280 }
12281}
12283//*************************************************************************************************
12284
12285} // namespace blaze
12286
12287#endif
Header file for the addition trait.
Header file for auxiliary alias declarations.
Header file for the alignment check function.
Header file for the alignment flag enumeration.
Header file for run time assertion macros.
Header file for kernel specific block sizes.
Header file for the blaze::checked and blaze::unchecked instances.
Constraints on the storage order of matrix types.
constexpr const DenseIterator< Type, AF > operator-(const DenseIterator< Type, AF > &it, ptrdiff_t inc) noexcept
Subtraction between a DenseIterator and an integral value.
Definition: DenseIterator.h:751
constexpr const DenseIterator< Type, AF > operator+(const DenseIterator< Type, AF > &it, ptrdiff_t inc) noexcept
Addition between a DenseIterator and an integral value.
Definition: DenseIterator.h:719
Header file for the EnableIf class template.
Header file for the HasMutableDataAccess type trait.
Header file for the HasSIMDAdd type trait.
Header file for the HasSIMDMult type trait.
Header file for the HasSIMDSub type trait.
Header file for the If class template.
Header file for the IsConst type trait.
Header file for the IsContiguous type trait.
Header file for the isDefault shim.
Header file for the IsDiagonal type trait.
Header file for the IsExpression type trait class.
Header file for the IsHermitian type trait.
Header file for the IsLower type trait.
Header file for the IsReference type trait.
Header file for the IsRestricted type trait.
Header file for the IsSIMDCombinable type trait.
Header file for the IsSparseMatrix type trait.
Header file for the IsStrictlyLower type trait.
Header file for the IsStrictlyUpper type trait.
Header file for the IsSymmetric type trait.
Header file for the IsTriangular type trait.
Header file for the IsUniLower type trait.
Header file for the IsUniUpper type trait.
Header file for the IsUpper type trait.
Header file for the MAYBE_UNUSED function template.
Constraint on the data type.
Header file for the prevMultiple shim.
Constraint on the data type.
Constraints on the storage order of matrix types.
Header file for all SIMD functionality.
Header file for the Schur product trait.
Header file for the subtraction trait.
Header file for the implementation of the SubmatrixData class template.
Header file for the submatrix trait.
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Initializer list type of the Blaze library.
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Header file for the implementation of a matrix representation of an initializer list.
Header file for the DenseMatrix base class.
Header file for the View base class.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:137
#define BLAZE_CONSTRAINT_MUST_NOT_BE_POINTER_TYPE(T)
Constraint on the data type.
Definition: Pointer.h:79
#define BLAZE_CONSTRAINT_MUST_BE_VECTORIZABLE_TYPE(T)
Constraint on the data type.
Definition: Vectorizable.h:61
#define BLAZE_CONSTRAINT_MUST_NOT_BE_REFERENCE_TYPE(T)
Constraint on the data type.
Definition: Reference.h:79
decltype(auto) transIf(const DenseMatrix< MT, SO > &dm)
Conditional calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:832
decltype(auto) min(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise minimum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1339
decltype(auto) max(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise maximum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1375
decltype(auto) ctrans(const DenseMatrix< MT, SO > &dm)
Returns the conjugate transpose matrix of dm.
Definition: DMatMapExpr.h:1501
decltype(auto) trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:766
MT::ElementType * data(DenseMatrix< MT, SO > &dm) noexcept
Low-level data access to the dense matrix elements.
Definition: DenseMatrix.h:182
auto operator+=(DenseMatrix< MT, SO > &mat, ST scalar) -> EnableIf_t< IsScalar_v< ST >, MT & >
Addition assignment operator for the addition of a dense matrix and a scalar value ( ).
Definition: DenseMatrix.h:386
size_t spacing(const DenseMatrix< MT, SO > &dm) noexcept
Returns the spacing between the beginning of two rows/columns.
Definition: DenseMatrix.h:265
decltype(auto) operator*(const DenseMatrix< MT1, false > &lhs, const DenseMatrix< MT2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:9640
auto operator-=(DenseMatrix< MT, SO > &mat, ST scalar) -> EnableIf_t< IsScalar_v< ST >, MT & >
Subtraction assignment operator for the subtraction of a dense matrix and a scalar value ( ).
Definition: DenseMatrix.h:448
bool isIntact(const DiagonalMatrix< MT, SO, DF > &m)
Returns whether the invariants of the given diagonal matrix are intact.
Definition: DiagonalMatrix.h:207
bool isDefault(const DiagonalMatrix< MT, SO, DF > &m)
Returns whether the given diagonal matrix is in default state.
Definition: DiagonalMatrix.h:169
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.
Definition: Symmetric.h:79
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.
Definition: RowMajorMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.
Definition: RequiresEvaluation.h:81
#define BLAZE_CONSTRAINT_MUST_NOT_BE_TRANSEXPR_TYPE(T)
Constraint on the data type.
Definition: TransExpr.h:81
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SUBMATRIX_TYPE(T)
Constraint on the data type.
Definition: Submatrix.h:81
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.
Definition: DenseMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_NOT_BE_COMPUTATION_TYPE(T)
Constraint on the data type.
Definition: Computation.h:81
#define BLAZE_CONSTRAINT_MUST_NOT_BE_UNITRIANGULAR_MATRIX_TYPE(T)
Constraint on the data type.
Definition: UniTriangular.h:81
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.
Definition: ColumnMajorMatrix.h:61
typename SubmatrixTrait< MT, CSAs... >::Type SubmatrixTrait_t
Auxiliary alias declaration for the SubmatrixTrait type trait.
Definition: SubmatrixTrait.h:145
constexpr bool HasSIMDSub_v
Auxiliary variable template for the HasSIMDSub type trait.
Definition: HasSIMDSub.h:187
constexpr bool IsSIMDCombinable_v
Auxiliary variable template for the IsSIMDCombinable type trait.
Definition: IsSIMDCombinable.h:137
constexpr bool IsDiagonal_v
Auxiliary variable template for the IsDiagonal type trait.
Definition: IsDiagonal.h:148
constexpr bool HasSIMDAdd_v
Auxiliary variable template for the HasSIMDAdd type trait.
Definition: HasSIMDAdd.h:187
constexpr bool HasSIMDMult_v
Auxiliary variable template for the HasSIMDMult type trait.
Definition: HasSIMDMult.h:188
constexpr bool IsContiguous_v
Auxiliary variable template for the IsContiguous type trait.
Definition: IsContiguous.h:144
BLAZE_ALWAYS_INLINE constexpr auto prevMultiple(T1 value, T2 factor) noexcept
Rounds down an integral value to the previous multiple of a given factor.
Definition: PrevMultiple.h:68
constexpr bool operator>(const NegativeAccuracy< A > &lhs, const T &rhs)
Greater-than comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:370
constexpr bool operator==(const NegativeAccuracy< A > &lhs, const T &rhs)
Equality comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:253
AlignmentFlag
Alignment flag for (un-)aligned vectors and matrices.
Definition: AlignmentFlag.h:63
constexpr bool operator<(const NegativeAccuracy< A > &lhs, const T &rhs)
Less-than comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:332
constexpr bool operator>=(const NegativeAccuracy< A > &, const T &rhs)
Greater-or-equal-than comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:446
constexpr bool operator!=(const NegativeAccuracy< A > &lhs, const T &rhs)
Inequality comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:293
constexpr bool operator<=(const NegativeAccuracy< A > &, const T &rhs)
Less-or-equal-than comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:408
@ unaligned
Flag for unaligned vectors and matrices.
Definition: AlignmentFlag.h:64
@ aligned
Flag for aligned vectors and matrices.
Definition: AlignmentFlag.h:65
constexpr void clear(Matrix< MT, SO > &matrix)
Clearing the given matrix.
Definition: Matrix.h:960
MT::ConstIterator cend(const Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:628
constexpr size_t rows(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of rows of the matrix.
Definition: Matrix.h:644
size_t nonZeros(const Matrix< MT, SO > &matrix)
Returns the total number of non-zero elements in the matrix.
Definition: Matrix.h:730
MT::ConstIterator cbegin(const Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:562
size_t capacity(const Matrix< MT, SO > &matrix) noexcept
Returns the maximum capacity of the matrix.
Definition: Matrix.h:692
constexpr void reset(Matrix< MT, SO > &matrix)
Resetting the given matrix.
Definition: Matrix.h:806
constexpr size_t columns(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of columns of the matrix.
Definition: Matrix.h:660
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:584
void ctranspose(Matrix< MT, SO > &matrix)
In-place conjugate transpose of the given matrix.
Definition: Matrix.h:1221
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:518
constexpr size_t size(const Matrix< MT, SO > &matrix) noexcept
Returns the total number of elements of the matrix.
Definition: Matrix.h:676
void transpose(Matrix< MT, SO > &matrix)
In-place transpose of the given matrix.
Definition: Matrix.h:1195
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:137
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.
Definition: Assert.h:101
#define BLAZE_USER_ASSERT(expr, msg)
Run time assertion macro for user checks.
Definition: Assert.h:117
BLAZE_ALWAYS_INLINE EnableIf_t< IsIntegral_v< T1 > &&HasSize_v< T1, 1UL > > storea(T1 *address, const SIMDi8< T2 > &value) noexcept
Aligned store of a vector of 1-byte integral values.
Definition: Storea.h:78
BLAZE_ALWAYS_INLINE EnableIf_t< IsIntegral_v< T1 > &&HasSize_v< T1, 1UL > > storeu(T1 *address, const SIMDi8< T2 > &value) noexcept
Unaligned store of a vector of 1-byte integral values.
Definition: Storeu.h:75
BLAZE_ALWAYS_INLINE EnableIf_t< IsIntegral_v< T1 > &&HasSize_v< T1, 1UL > > stream(T1 *address, const SIMDi8< T2 > &value) noexcept
Aligned, non-temporal store of a vector of 1-byte integral values.
Definition: Stream.h:74
BLAZE_ALWAYS_INLINE const EnableIf_t< IsIntegral_v< T > &&HasSize_v< T, 1UL >, If_t< IsSigned_v< T >, SIMDint8, SIMDuint8 > > loadu(const T *address) noexcept
Loads a vector of 1-byte integral values.
Definition: Loadu.h:76
BLAZE_ALWAYS_INLINE const EnableIf_t< IsIntegral_v< T > &&HasSize_v< T, 1UL >, If_t< IsSigned_v< T >, SIMDint8, SIMDuint8 > > loada(const T *address) noexcept
Loads a vector of 1-byte integral values.
Definition: Loada.h:79
auto smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:162
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:100
auto smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:194
auto smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:131
#define BLAZE_ALWAYS_INLINE
Platform dependent setup of an enforced inline keyword.
Definition: Inline.h:85
typename EnableIf< Condition, T >::Type EnableIf_t
Auxiliary type for the EnableIf class template.
Definition: EnableIf.h:138
constexpr void MAYBE_UNUSED(const Args &...)
Suppression of unused parameter warnings.
Definition: MaybeUnused.h:81
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.
Definition: Exception.h:331
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.
Definition: Exception.h:235
BLAZE_ALWAYS_INLINE bool checkAlignment(const T *address)
Checks the alignment of the given address.
Definition: AlignmentCheck.h:68
#define BLAZE_THROW_LOGIC_ERROR(MESSAGE)
Macro for the emission of a std::logic_error exception.
Definition: Exception.h:187
typename EnableIf<!Condition, T >::Type DisableIf_t
Auxiliary type for the EnableIf class template.
Definition: EnableIf.h:175
constexpr bool isChecked(const Ts &... args)
Extracting blaze::Check arguments from a given list of arguments.
Definition: Check.h:225
Header file for the exception macros of the math module.
Header file for the extended initializer_list functionality.
Header file for the matrix storage order types.
Header file for the clear shim.
Header file for the reset shim.
Header file for the cache size of the target architecture.
System settings for the inline keywords.
System settings for performance optimizations.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
Header file for the RequiresEvaluation type trait.
Header file for basic type definitions.
Header file for the generic max algorithm.
Header file for the generic min algorithm.
Header file for the implementation of the Submatrix base template.