Blaze  3.6
Dense.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_VIEWS_SUBMATRIX_DENSE_H_
36 #define _BLAZE_MATH_VIEWS_SUBMATRIX_DENSE_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <algorithm>
44 #include <iterator>
45 #include <blaze/math/Aliases.h>
57 #include <blaze/math/Exception.h>
61 #include <blaze/math/shims/Clear.h>
63 #include <blaze/math/shims/Reset.h>
64 #include <blaze/math/SIMD.h>
90 #include <blaze/math/views/Check.h>
93 #include <blaze/system/Blocking.h>
94 #include <blaze/system/CacheSize.h>
95 #include <blaze/system/Inline.h>
101 #include <blaze/util/Assert.h>
105 #include <blaze/util/DisableIf.h>
106 #include <blaze/util/EnableIf.h>
107 #include <blaze/util/MaybeUnused.h>
108 #include <blaze/util/mpl/If.h>
109 #include <blaze/util/TypeList.h>
110 #include <blaze/util/Types.h>
113 
114 
115 namespace blaze {
116 
117 //=================================================================================================
118 //
119 // CLASS TEMPLATE SPECIALIZATION FOR UNALIGNED ROW-MAJOR DENSE SUBMATRICES
120 //
121 //=================================================================================================
122 
123 //*************************************************************************************************
131 template< typename MT // Type of the dense matrix
132  , size_t... CSAs > // Compile time submatrix arguments
133 class Submatrix<MT,unaligned,false,true,CSAs...>
134  : public View< DenseMatrix< Submatrix<MT,unaligned,false,true,CSAs...>, false > >
135  , private SubmatrixData<CSAs...>
136 {
137  private:
138  //**Type definitions****************************************************************************
139  using DataType = SubmatrixData<CSAs...>;
140  using Operand = If_t< IsExpression_v<MT>, MT, MT& >;
141  //**********************************************************************************************
142 
143  //**********************************************************************************************
145  template< typename MT1, typename MT2 >
146  static constexpr bool EnforceEvaluation_v =
147  ( IsRestricted_v<MT1> && RequiresEvaluation_v<MT2> );
148  //**********************************************************************************************
149 
150  public:
151  //**Type definitions****************************************************************************
153  using This = Submatrix<MT,unaligned,false,true,CSAs...>;
154 
155  using BaseType = DenseMatrix<This,false>;
156  using ViewedType = MT;
157  using ResultType = SubmatrixTrait_t<MT,CSAs...>;
158  using OppositeType = OppositeType_t<ResultType>;
159  using TransposeType = TransposeType_t<ResultType>;
160  using ElementType = ElementType_t<MT>;
161  using SIMDType = SIMDTrait_t<ElementType>;
162  using ReturnType = ReturnType_t<MT>;
163  using CompositeType = const Submatrix&;
164 
166  using ConstReference = ConstReference_t<MT>;
167 
169  using Reference = If_t< IsConst_v<MT>, ConstReference, Reference_t<MT> >;
170 
172  using ConstPointer = ConstPointer_t<MT>;
173 
175  using Pointer = If_t< IsConst_v<MT> || !HasMutableDataAccess_v<MT>, ConstPointer, Pointer_t<MT> >;
176  //**********************************************************************************************
177 
178  //**SubmatrixIterator class definition**********************************************************
181  template< typename IteratorType > // Type of the dense matrix iterator
182  class SubmatrixIterator
183  {
184  public:
185  //**Type definitions*************************************************************************
187  using IteratorCategory = typename std::iterator_traits<IteratorType>::iterator_category;
188 
190  using ValueType = typename std::iterator_traits<IteratorType>::value_type;
191 
193  using PointerType = typename std::iterator_traits<IteratorType>::pointer;
194 
196  using ReferenceType = typename std::iterator_traits<IteratorType>::reference;
197 
199  using DifferenceType = typename std::iterator_traits<IteratorType>::difference_type;
200 
201  // STL iterator requirements
202  using iterator_category = IteratorCategory;
203  using value_type = ValueType;
204  using pointer = PointerType;
205  using reference = ReferenceType;
206  using difference_type = DifferenceType;
207  //*******************************************************************************************
208 
209  //**Constructor******************************************************************************
212  inline SubmatrixIterator()
213  : iterator_ ( ) // Iterator to the current submatrix element
214  , isAligned_( false ) // Memory alignment flag
215  {}
216  //*******************************************************************************************
217 
218  //**Constructor******************************************************************************
224  inline SubmatrixIterator( IteratorType iterator, bool isMemoryAligned )
225  : iterator_ ( iterator ) // Iterator to the current submatrix element
226  , isAligned_( isMemoryAligned ) // Memory alignment flag
227  {}
228  //*******************************************************************************************
229 
230  //**Constructor******************************************************************************
235  template< typename IteratorType2 >
236  inline SubmatrixIterator( const SubmatrixIterator<IteratorType2>& it )
237  : iterator_ ( it.base() ) // Iterator to the current submatrix element
238  , isAligned_( it.isAligned() ) // Memory alignment flag
239  {}
240  //*******************************************************************************************
241 
242  //**Addition assignment operator*************************************************************
248  inline SubmatrixIterator& operator+=( size_t inc ) {
249  iterator_ += inc;
250  return *this;
251  }
252  //*******************************************************************************************
253 
254  //**Subtraction assignment operator**********************************************************
260  inline SubmatrixIterator& operator-=( size_t dec ) {
261  iterator_ -= dec;
262  return *this;
263  }
264  //*******************************************************************************************
265 
266  //**Prefix increment operator****************************************************************
271  inline SubmatrixIterator& operator++() {
272  ++iterator_;
273  return *this;
274  }
275  //*******************************************************************************************
276 
277  //**Postfix increment operator***************************************************************
282  inline const SubmatrixIterator operator++( int ) {
283  return SubmatrixIterator( iterator_++, isAligned_ );
284  }
285  //*******************************************************************************************
286 
287  //**Prefix decrement operator****************************************************************
292  inline SubmatrixIterator& operator--() {
293  --iterator_;
294  return *this;
295  }
296  //*******************************************************************************************
297 
298  //**Postfix decrement operator***************************************************************
303  inline const SubmatrixIterator operator--( int ) {
304  return SubmatrixIterator( iterator_--, isAligned_ );
305  }
306  //*******************************************************************************************
307 
308  //**Element access operator******************************************************************
313  inline ReferenceType operator*() const {
314  return *iterator_;
315  }
316  //*******************************************************************************************
317 
318  //**Element access operator******************************************************************
323  inline IteratorType operator->() const {
324  return iterator_;
325  }
326  //*******************************************************************************************
327 
328  //**Load function****************************************************************************
338  inline SIMDType load() const noexcept {
339  if( isAligned_ )
340  return loada();
341  else
342  return loadu();
343  }
344  //*******************************************************************************************
345 
346  //**Loada function***************************************************************************
356  inline SIMDType loada() const noexcept {
357  return iterator_.loada();
358  }
359  //*******************************************************************************************
360 
361  //**Loadu function***************************************************************************
371  inline SIMDType loadu() const noexcept {
372  return iterator_.loadu();
373  }
374  //*******************************************************************************************
375 
376  //**Store function***************************************************************************
387  inline void store( const SIMDType& value ) const {
388  if( isAligned_ ) {
389  storea( value );
390  }
391  else {
392  storeu( value );
393  }
394  }
395  //*******************************************************************************************
396 
397  //**Storea function**************************************************************************
408  inline void storea( const SIMDType& value ) const {
409  iterator_.storea( value );
410  }
411  //*******************************************************************************************
412 
413  //**Storeu function**************************************************************************
424  inline void storeu( const SIMDType& value ) const {
425  iterator_.storeu( value );
426  }
427  //*******************************************************************************************
428 
429  //**Stream function**************************************************************************
440  inline void stream( const SIMDType& value ) const {
441  iterator_.stream( value );
442  }
443  //*******************************************************************************************
444 
445  //**Equality operator************************************************************************
451  inline bool operator==( const SubmatrixIterator& rhs ) const {
452  return iterator_ == rhs.iterator_;
453  }
454  //*******************************************************************************************
455 
456  //**Inequality operator**********************************************************************
462  inline bool operator!=( const SubmatrixIterator& rhs ) const {
463  return iterator_ != rhs.iterator_;
464  }
465  //*******************************************************************************************
466 
467  //**Less-than operator***********************************************************************
473  inline bool operator<( const SubmatrixIterator& rhs ) const {
474  return iterator_ < rhs.iterator_;
475  }
476  //*******************************************************************************************
477 
478  //**Greater-than operator********************************************************************
484  inline bool operator>( const SubmatrixIterator& rhs ) const {
485  return iterator_ > rhs.iterator_;
486  }
487  //*******************************************************************************************
488 
489  //**Less-or-equal-than operator**************************************************************
495  inline bool operator<=( const SubmatrixIterator& rhs ) const {
496  return iterator_ <= rhs.iterator_;
497  }
498  //*******************************************************************************************
499 
500  //**Greater-or-equal-than operator***********************************************************
506  inline bool operator>=( const SubmatrixIterator& rhs ) const {
507  return iterator_ >= rhs.iterator_;
508  }
509  //*******************************************************************************************
510 
511  //**Subtraction operator*********************************************************************
517  inline DifferenceType operator-( const SubmatrixIterator& rhs ) const {
518  return iterator_ - rhs.iterator_;
519  }
520  //*******************************************************************************************
521 
522  //**Addition operator************************************************************************
529  friend inline const SubmatrixIterator operator+( const SubmatrixIterator& it, size_t inc ) {
530  return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
531  }
532  //*******************************************************************************************
533 
534  //**Addition operator************************************************************************
541  friend inline const SubmatrixIterator operator+( size_t inc, const SubmatrixIterator& it ) {
542  return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
543  }
544  //*******************************************************************************************
545 
546  //**Subtraction operator*********************************************************************
553  friend inline const SubmatrixIterator operator-( const SubmatrixIterator& it, size_t dec ) {
554  return SubmatrixIterator( it.iterator_ - dec, it.isAligned_ );
555  }
556  //*******************************************************************************************
557 
558  //**Base function****************************************************************************
563  inline IteratorType base() const {
564  return iterator_;
565  }
566  //*******************************************************************************************
567 
568  //**IsAligned function***********************************************************************
573  inline bool isAligned() const noexcept {
574  return isAligned_;
575  }
576  //*******************************************************************************************
577 
578  private:
579  //**Member variables*************************************************************************
580  IteratorType iterator_;
581  bool isAligned_;
582  //*******************************************************************************************
583  };
584  //**********************************************************************************************
585 
586  //**Type definitions****************************************************************************
588  using ConstIterator = SubmatrixIterator< ConstIterator_t<MT> >;
589 
591  using Iterator = If_t< IsConst_v<MT>, ConstIterator, SubmatrixIterator< Iterator_t<MT> > >;
592  //**********************************************************************************************
593 
594  //**Compilation flags***************************************************************************
596  static constexpr bool simdEnabled = MT::simdEnabled;
597 
599  static constexpr bool smpAssignable = MT::smpAssignable;
600 
602  static constexpr bool compileTimeArgs = DataType::compileTimeArgs;
603  //**********************************************************************************************
604 
605  //**Constructors********************************************************************************
608  template< typename... RSAs >
609  explicit inline Submatrix( MT& matrix, RSAs... args );
610 
611  Submatrix( const Submatrix& ) = default;
613  //**********************************************************************************************
614 
615  //**Destructor**********************************************************************************
618  ~Submatrix() = default;
620  //**********************************************************************************************
621 
622  //**Data access functions***********************************************************************
625  inline Reference operator()( size_t i, size_t j );
626  inline ConstReference operator()( size_t i, size_t j ) const;
627  inline Reference at( size_t i, size_t j );
628  inline ConstReference at( size_t i, size_t j ) const;
629  inline Pointer data () noexcept;
630  inline ConstPointer data () const noexcept;
631  inline Pointer data ( size_t i ) noexcept;
632  inline ConstPointer data ( size_t i ) const noexcept;
633  inline Iterator begin ( size_t i );
634  inline ConstIterator begin ( size_t i ) const;
635  inline ConstIterator cbegin( size_t i ) const;
636  inline Iterator end ( size_t i );
637  inline ConstIterator end ( size_t i ) const;
638  inline ConstIterator cend ( size_t i ) const;
640  //**********************************************************************************************
641 
642  //**Assignment operators************************************************************************
645  inline Submatrix& operator=( const ElementType& rhs );
646  inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
647  inline Submatrix& operator=( const Submatrix& rhs );
648 
649  template< typename MT2, bool SO2 >
650  inline Submatrix& operator=( const Matrix<MT2,SO2>& rhs );
651 
652  template< typename MT2, bool SO2 >
653  inline auto operator+=( const Matrix<MT2,SO2>& rhs )
654  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
655 
656  template< typename MT2, bool SO2 >
657  inline auto operator+=( const Matrix<MT2,SO2>& rhs )
658  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
659 
660  template< typename MT2, bool SO2 >
661  inline auto operator-=( const Matrix<MT2,SO2>& rhs )
662  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
663 
664  template< typename MT2, bool SO2 >
665  inline auto operator-=( const Matrix<MT2,SO2>& rhs )
666  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
667 
668  template< typename MT2, bool SO2 >
669  inline auto operator%=( const Matrix<MT2,SO2>& rhs )
670  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
671 
672  template< typename MT2, bool SO2 >
673  inline auto operator%=( const Matrix<MT2,SO2>& rhs )
674  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
676  //**********************************************************************************************
677 
678  //**Utility functions***************************************************************************
681  using DataType::row;
682  using DataType::column;
683  using DataType::rows;
684  using DataType::columns;
685 
686  inline MT& operand() noexcept;
687  inline const MT& operand() const noexcept;
688 
689  inline size_t spacing() const noexcept;
690  inline size_t capacity() const noexcept;
691  inline size_t capacity( size_t i ) const noexcept;
692  inline size_t nonZeros() const;
693  inline size_t nonZeros( size_t i ) const;
694  inline void reset();
695  inline void reset( size_t i );
697  //**********************************************************************************************
698 
699  //**Numeric functions***************************************************************************
702  inline Submatrix& transpose();
703  inline Submatrix& ctranspose();
704 
705  template< typename Other > inline Submatrix& scale( const Other& scalar );
707  //**********************************************************************************************
708 
709  private:
710  //**********************************************************************************************
712  template< typename MT2 >
713  static constexpr bool VectorizedAssign_v =
714  ( useOptimizedKernels &&
715  simdEnabled && MT2::simdEnabled &&
716  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > );
717  //**********************************************************************************************
718 
719  //**********************************************************************************************
721  template< typename MT2 >
722  static constexpr bool VectorizedAddAssign_v =
723  ( useOptimizedKernels &&
724  simdEnabled && MT2::simdEnabled &&
725  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > &&
726  HasSIMDAdd_v< ElementType, ElementType_t<MT2> > &&
727  !IsDiagonal_v<MT2> );
728  //**********************************************************************************************
729 
730  //**********************************************************************************************
732  template< typename MT2 >
733  static constexpr bool VectorizedSubAssign_v =
734  ( useOptimizedKernels &&
735  simdEnabled && MT2::simdEnabled &&
736  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > &&
737  HasSIMDSub_v< ElementType, ElementType_t<MT2> > &&
738  !IsDiagonal_v<MT2> );
739  //**********************************************************************************************
740 
741  //**********************************************************************************************
743  template< typename MT2 >
744  static constexpr bool VectorizedSchurAssign_v =
745  ( useOptimizedKernels &&
746  simdEnabled && MT2::simdEnabled &&
747  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > &&
748  HasSIMDMult_v< ElementType, ElementType_t<MT2> > );
749  //**********************************************************************************************
750 
751  //**SIMD properties*****************************************************************************
753  static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
754  //**********************************************************************************************
755 
756  public:
757  //**Expression template evaluation functions****************************************************
760  template< typename Other >
761  inline bool canAlias( const Other* alias ) const noexcept;
762 
763  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
764  inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
765 
766  template< typename Other >
767  inline bool isAliased( const Other* alias ) const noexcept;
768 
769  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
770  inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
771 
772  inline bool isAligned () const noexcept;
773  inline bool canSMPAssign() const noexcept;
774 
775  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
776  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
777  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
778 
779  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
780  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
781  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
782  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
783 
784  template< typename MT2 >
785  inline auto assign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT2> >;
786 
787  template< typename MT2 >
788  inline auto assign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT2> >;
789 
790  template< typename MT2 > inline void assign( const DenseMatrix<MT2,true>& rhs );
791  template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
792  template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
793 
794  template< typename MT2 >
795  inline auto addAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT2> >;
796 
797  template< typename MT2 >
798  inline auto addAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT2> >;
799 
800  template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,true>& rhs );
801  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
802  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
803 
804  template< typename MT2 >
805  inline auto subAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT2> >;
806 
807  template< typename MT2 >
808  inline auto subAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT2> >;
809 
810  template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,true>& rhs );
811  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
812  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
813 
814  template< typename MT2 >
815  inline auto schurAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT2> >;
816 
817  template< typename MT2 >
818  inline auto schurAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT2> >;
819 
820  template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,true>& rhs );
821  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
822  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
824  //**********************************************************************************************
825 
826  private:
827  //**Utility functions***************************************************************************
830  inline bool hasOverlap() const noexcept;
832  //**********************************************************************************************
833 
834  //**Member variables****************************************************************************
837  Operand matrix_;
838  const bool isAligned_;
839 
846  //**********************************************************************************************
847 
848  //**Friend declarations*************************************************************************
849  template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
850  //**********************************************************************************************
851 
852  //**Compile time checks*************************************************************************
860  //**********************************************************************************************
861 };
863 //*************************************************************************************************
864 
865 
866 
867 
868 //=================================================================================================
869 //
870 // CONSTRUCTORS
871 //
872 //=================================================================================================
873 
874 //*************************************************************************************************
887 template< typename MT // Type of the dense matrix
888  , size_t... CSAs > // Compile time submatrix arguments
889 template< typename... RSAs > // Runtime submatrix arguments
890 inline Submatrix<MT,unaligned,false,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
891  : DataType ( args... ) // Base class initialization
892  , matrix_ ( matrix ) // The matrix containing the submatrix
893  , isAligned_( simdEnabled && IsContiguous_v<MT> &&
894  matrix.data() != nullptr && checkAlignment( data() ) &&
895  ( rows() < 2UL || ( matrix.spacing() & size_t(-SIMDSIZE) ) == 0UL ) )
896 {
897  if( !Contains_v< TypeList<RSAs...>, Unchecked > ) {
898  if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
899  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
900  }
901  }
902  else {
903  BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
904  BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
905  }
906 }
908 //*************************************************************************************************
909 
910 
911 
912 
913 //=================================================================================================
914 //
915 // DATA ACCESS FUNCTIONS
916 //
917 //=================================================================================================
918 
919 //*************************************************************************************************
930 template< typename MT // Type of the dense matrix
931  , size_t... CSAs > // Compile time submatrix arguments
932 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Reference
933  Submatrix<MT,unaligned,false,true,CSAs...>::operator()( size_t i, size_t j )
934 {
935  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
936  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
937 
938  return matrix_(row()+i,column()+j);
939 }
941 //*************************************************************************************************
942 
943 
944 //*************************************************************************************************
955 template< typename MT // Type of the dense matrix
956  , size_t... CSAs > // Compile time submatrix arguments
957 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstReference
958  Submatrix<MT,unaligned,false,true,CSAs...>::operator()( size_t i, size_t j ) const
959 {
960  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
961  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
962 
963  return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
964 }
966 //*************************************************************************************************
967 
968 
969 //*************************************************************************************************
981 template< typename MT // Type of the dense matrix
982  , size_t... CSAs > // Compile time submatrix arguments
983 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Reference
984  Submatrix<MT,unaligned,false,true,CSAs...>::at( size_t i, size_t j )
985 {
986  if( i >= rows() ) {
987  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
988  }
989  if( j >= columns() ) {
990  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
991  }
992  return (*this)(i,j);
993 }
995 //*************************************************************************************************
996 
997 
998 //*************************************************************************************************
1010 template< typename MT // Type of the dense matrix
1011  , size_t... CSAs > // Compile time submatrix arguments
1012 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstReference
1013  Submatrix<MT,unaligned,false,true,CSAs...>::at( size_t i, size_t j ) const
1014 {
1015  if( i >= rows() ) {
1016  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
1017  }
1018  if( j >= columns() ) {
1019  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
1020  }
1021  return (*this)(i,j);
1022 }
1024 //*************************************************************************************************
1025 
1026 
1027 //*************************************************************************************************
1037 template< typename MT // Type of the dense matrix
1038  , size_t... CSAs > // Compile time submatrix arguments
1039 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Pointer
1041 {
1042  return matrix_.data() + row()*spacing() + column();
1043 }
1045 //*************************************************************************************************
1046 
1047 
1048 //*************************************************************************************************
1058 template< typename MT // Type of the dense matrix
1059  , size_t... CSAs > // Compile time submatrix arguments
1060 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstPointer
1062 {
1063  return matrix_.data() + row()*spacing() + column();
1064 }
1066 //*************************************************************************************************
1067 
1068 
1069 //*************************************************************************************************
1078 template< typename MT // Type of the dense matrix
1079  , size_t... CSAs > // Compile time submatrix arguments
1080 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Pointer
1082 {
1083  return matrix_.data() + (row()+i)*spacing() + column();
1084 }
1086 //*************************************************************************************************
1087 
1088 
1089 //*************************************************************************************************
1098 template< typename MT // Type of the dense matrix
1099  , size_t... CSAs > // Compile time submatrix arguments
1100 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstPointer
1101  Submatrix<MT,unaligned,false,true,CSAs...>::data( size_t i ) const noexcept
1102 {
1103  return matrix_.data() + (row()+i)*spacing() + column();
1104 }
1106 //*************************************************************************************************
1107 
1108 
1109 //*************************************************************************************************
1121 template< typename MT // Type of the dense matrix
1122  , size_t... CSAs > // Compile time submatrix arguments
1123 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Iterator
1125 {
1126  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1127  return Iterator( matrix_.begin( row() + i ) + column(), isAligned_ );
1128 }
1130 //*************************************************************************************************
1131 
1132 
1133 //*************************************************************************************************
1145 template< typename MT // Type of the dense matrix
1146  , size_t... CSAs > // Compile time submatrix arguments
1147 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1149 {
1150  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1151  return ConstIterator( matrix_.cbegin( row() + i ) + column(), isAligned_ );
1152 }
1154 //*************************************************************************************************
1155 
1156 
1157 //*************************************************************************************************
1169 template< typename MT // Type of the dense matrix
1170  , size_t... CSAs > // Compile time submatrix arguments
1171 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1173 {
1174  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1175  return ConstIterator( matrix_.cbegin( row() + i ) + column(), isAligned_ );
1176 }
1178 //*************************************************************************************************
1179 
1180 
1181 //*************************************************************************************************
1193 template< typename MT // Type of the dense matrix
1194  , size_t... CSAs > // Compile time submatrix arguments
1195 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Iterator
1197 {
1198  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1199  return Iterator( matrix_.begin( row() + i ) + column() + columns(), isAligned_ );
1200 }
1202 //*************************************************************************************************
1203 
1204 
1205 //*************************************************************************************************
1217 template< typename MT // Type of the dense matrix
1218  , size_t... CSAs > // Compile time submatrix arguments
1219 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1221 {
1222  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1223  return ConstIterator( matrix_.cbegin( row() + i ) + column() + columns(), isAligned_ );
1224 }
1226 //*************************************************************************************************
1227 
1228 
1229 //*************************************************************************************************
1241 template< typename MT // Type of the dense matrix
1242  , size_t... CSAs > // Compile time submatrix arguments
1243 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1245 {
1246  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1247  return ConstIterator( matrix_.cbegin( row() + i ) + column() + columns(), isAligned_ );
1248 }
1250 //*************************************************************************************************
1251 
1252 
1253 
1254 
1255 //=================================================================================================
1256 //
1257 // ASSIGNMENT OPERATORS
1258 //
1259 //=================================================================================================
1260 
1261 //*************************************************************************************************
1272 template< typename MT // Type of the dense matrix
1273  , size_t... CSAs > // Compile time submatrix arguments
1274 inline Submatrix<MT,unaligned,false,true,CSAs...>&
1275  Submatrix<MT,unaligned,false,true,CSAs...>::operator=( const ElementType& rhs )
1276 {
1277  const size_t iend( row() + rows() );
1278  decltype(auto) left( derestrict( matrix_ ) );
1279 
1280  for( size_t i=row(); i<iend; ++i )
1281  {
1282  const size_t jbegin( ( IsUpper_v<MT> )
1283  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
1284  ?( max( i+1UL, column() ) )
1285  :( max( i, column() ) ) )
1286  :( column() ) );
1287  const size_t jend ( ( IsLower_v<MT> )
1288  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
1289  ?( min( i, column()+columns() ) )
1290  :( min( i+1UL, column()+columns() ) ) )
1291  :( column()+columns() ) );
1292 
1293  for( size_t j=jbegin; j<jend; ++j ) {
1294  if( !IsRestricted_v<MT> || IsTriangular_v<MT> || trySet( matrix_, i, j, rhs ) )
1295  left(i,j) = rhs;
1296  }
1297  }
1298 
1299  return *this;
1300 }
1302 //*************************************************************************************************
1303 
1304 
1305 //*************************************************************************************************
1321 template< typename MT // Type of the dense matrix
1322  , size_t... CSAs > // Compile time submatrix arguments
1323 inline Submatrix<MT,unaligned,false,true,CSAs...>&
1324  Submatrix<MT,unaligned,false,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
1325 {
1326  if( list.size() != rows() ) {
1327  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
1328  }
1329 
1330  if( IsRestricted_v<MT> ) {
1331  const InitializerMatrix<ElementType> tmp( list, columns() );
1332  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1333  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1334  }
1335  }
1336 
1337  decltype(auto) left( derestrict( *this ) );
1338  size_t i( 0UL );
1339 
1340  for( const auto& rowList : list ) {
1341  std::fill( std::copy( rowList.begin(), rowList.end(), left.begin(i) ), left.end(i), ElementType() );
1342  ++i;
1343  }
1344 
1345  return *this;
1346 }
1348 //*************************************************************************************************
1349 
1350 
1351 //*************************************************************************************************
1366 template< typename MT // Type of the dense matrix
1367  , size_t... CSAs > // Compile time submatrix arguments
1368 inline Submatrix<MT,unaligned,false,true,CSAs...>&
1369  Submatrix<MT,unaligned,false,true,CSAs...>::operator=( const Submatrix& rhs )
1370 {
1373 
1374  if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
1375  return *this;
1376 
1377  if( rows() != rhs.rows() || columns() != rhs.columns() ) {
1378  BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
1379  }
1380 
1381  if( !tryAssign( matrix_, rhs, row(), column() ) ) {
1382  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1383  }
1384 
1385  decltype(auto) left( derestrict( *this ) );
1386 
1387  if( rhs.canAlias( &matrix_ ) ) {
1388  const ResultType tmp( rhs );
1389  smpAssign( left, tmp );
1390  }
1391  else {
1392  smpAssign( left, rhs );
1393  }
1394 
1395  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1396 
1397  return *this;
1398 }
1400 //*************************************************************************************************
1401 
1402 
1403 //*************************************************************************************************
1418 template< typename MT // Type of the dense matrix
1419  , size_t... CSAs > // Compile time submatrix arguments
1420 template< typename MT2 // Type of the right-hand side matrix
1421  , bool SO2 > // Storage order of the right-hand side matrix
1422 inline Submatrix<MT,unaligned,false,true,CSAs...>&
1423  Submatrix<MT,unaligned,false,true,CSAs...>::operator=( const Matrix<MT2,SO2>& rhs )
1424 {
1425  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1426 
1427  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1428  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1429  }
1430 
1431  using Right = If_t< IsRestricted_v<MT>, CompositeType_t<MT2>, const MT2& >;
1432  Right right( ~rhs );
1433 
1434  if( !tryAssign( matrix_, right, row(), column() ) ) {
1435  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1436  }
1437 
1438  decltype(auto) left( derestrict( *this ) );
1439 
1440  if( IsReference_v<Right> && right.canAlias( &matrix_ ) ) {
1441  const ResultType_t<MT2> tmp( right );
1442  if( IsSparseMatrix_v<MT2> )
1443  reset();
1444  smpAssign( left, tmp );
1445  }
1446  else {
1447  if( IsSparseMatrix_v<MT2> )
1448  reset();
1449  smpAssign( left, right );
1450  }
1451 
1452  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1453 
1454  return *this;
1455 }
1457 //*************************************************************************************************
1458 
1459 
1460 //*************************************************************************************************
1474 template< typename MT // Type of the dense matrix
1475  , size_t... CSAs > // Compile time submatrix arguments
1476 template< typename MT2 // Type of the right-hand side matrix
1477  , bool SO2 > // Storage order of the right-hand side matrix
1478 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO2>& rhs )
1479  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1480 {
1483  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1484 
1485  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
1486 
1489 
1490  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1491  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1492  }
1493 
1494  if( !tryAddAssign( matrix_, ~rhs, row(), column() ) ) {
1495  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1496  }
1497 
1498  decltype(auto) left( derestrict( *this ) );
1499 
1500  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
1501  (~rhs).canAlias( &matrix_ ) ) {
1502  const AddType tmp( *this + (~rhs) );
1503  smpAssign( left, tmp );
1504  }
1505  else {
1506  smpAddAssign( left, ~rhs );
1507  }
1508 
1509  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1510 
1511  return *this;
1512 }
1514 //*************************************************************************************************
1515 
1516 
1517 //*************************************************************************************************
1531 template< typename MT // Type of the dense matrix
1532  , size_t... CSAs > // Compile time submatrix arguments
1533 template< typename MT2 // Type of the right-hand side matrix
1534  , bool SO2 > // Storage order of the right-hand side matrix
1535 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO2>& rhs )
1536  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1537 {
1540  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1541 
1542  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
1543 
1546 
1547  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1548  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1549  }
1550 
1551  const AddType tmp( *this + (~rhs) );
1552 
1553  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1554  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1555  }
1556 
1557  decltype(auto) left( derestrict( *this ) );
1558 
1559  smpAssign( left, tmp );
1560 
1561  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1562 
1563  return *this;
1564 }
1566 //*************************************************************************************************
1567 
1568 
1569 //*************************************************************************************************
1583 template< typename MT // Type of the dense matrix
1584  , size_t... CSAs > // Compile time submatrix arguments
1585 template< typename MT2 // Type of the right-hand side matrix
1586  , bool SO2 > // Storage order of the right-hand side matrix
1587 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO2>& rhs )
1588  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1589 {
1592  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1593 
1594  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
1595 
1598 
1599  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1600  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1601  }
1602 
1603  if( !trySubAssign( matrix_, ~rhs, row(), column() ) ) {
1604  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1605  }
1606 
1607  decltype(auto) left( derestrict( *this ) );
1608 
1609  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
1610  (~rhs).canAlias( &matrix_ ) ) {
1611  const SubType tmp( *this - (~rhs ) );
1612  smpAssign( left, tmp );
1613  }
1614  else {
1615  smpSubAssign( left, ~rhs );
1616  }
1617 
1618  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1619 
1620  return *this;
1621 }
1623 //*************************************************************************************************
1624 
1625 
1626 //*************************************************************************************************
1640 template< typename MT // Type of the dense matrix
1641  , size_t... CSAs > // Compile time submatrix arguments
1642 template< typename MT2 // Type of the right-hand side matrix
1643  , bool SO2 > // Storage order of the right-hand side matrix
1644 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO2>& rhs )
1645  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1646 {
1649  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1650 
1651  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
1652 
1655 
1656  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1657  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1658  }
1659 
1660  const SubType tmp( *this - (~rhs) );
1661 
1662  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1663  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1664  }
1665 
1666  decltype(auto) left( derestrict( *this ) );
1667 
1668  smpAssign( left, tmp );
1669 
1670  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1671 
1672  return *this;
1673 }
1675 //*************************************************************************************************
1676 
1677 
1678 //*************************************************************************************************
1692 template< typename MT // Type of the dense matrix
1693  , size_t... CSAs > // Compile time submatrix arguments
1694 template< typename MT2 // Type of the right-hand side matrix
1695  , bool SO2 > // Storage order of the right-hand side matrix
1696 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO2>& rhs )
1697  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1698 {
1701  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1702 
1703  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
1704 
1706 
1707  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1708  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1709  }
1710 
1711  if( !trySchurAssign( matrix_, ~rhs, row(), column() ) ) {
1712  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1713  }
1714 
1715  decltype(auto) left( derestrict( *this ) );
1716 
1717  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
1718  (~rhs).canAlias( &matrix_ ) ) {
1719  const SchurType tmp( *this % (~rhs) );
1720  if( IsSparseMatrix_v<SchurType> )
1721  reset();
1722  smpAssign( left, tmp );
1723  }
1724  else {
1725  smpSchurAssign( left, ~rhs );
1726  }
1727 
1728  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1729 
1730  return *this;
1731 }
1733 //*************************************************************************************************
1734 
1735 
1736 //*************************************************************************************************
1750 template< typename MT // Type of the dense matrix
1751  , size_t... CSAs > // Compile time submatrix arguments
1752 template< typename MT2 // Type of the right-hand side matrix
1753  , bool SO2 > // Storage order of the right-hand side matrix
1754 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO2>& rhs )
1755  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1756 {
1759  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1760 
1761  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
1762 
1764 
1765  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1766  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1767  }
1768 
1769  const SchurType tmp( *this % (~rhs) );
1770 
1771  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1772  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1773  }
1774 
1775  decltype(auto) left( derestrict( *this ) );
1776 
1777  if( IsSparseMatrix_v<SchurType> ) {
1778  reset();
1779  }
1780 
1781  smpAssign( left, tmp );
1782 
1783  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1784 
1785  return *this;
1786 }
1788 //*************************************************************************************************
1789 
1790 
1791 
1792 
1793 //=================================================================================================
1794 //
1795 // UTILITY FUNCTIONS
1796 //
1797 //=================================================================================================
1798 
1799 //*************************************************************************************************
1805 template< typename MT // Type of the dense matrix
1806  , size_t... CSAs > // Compile time submatrix arguments
1807 inline MT& Submatrix<MT,unaligned,false,true,CSAs...>::operand() noexcept
1808 {
1809  return matrix_;
1810 }
1812 //*************************************************************************************************
1813 
1814 
1815 //*************************************************************************************************
1821 template< typename MT // Type of the dense matrix
1822  , size_t... CSAs > // Compile time submatrix arguments
1823 inline const MT& Submatrix<MT,unaligned,false,true,CSAs...>::operand() const noexcept
1824 {
1825  return matrix_;
1826 }
1828 //*************************************************************************************************
1829 
1830 
1831 //*************************************************************************************************
1842 template< typename MT // Type of the dense matrix
1843  , size_t... CSAs > // Compile time submatrix arguments
1844 inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::spacing() const noexcept
1845 {
1846  return matrix_.spacing();
1847 }
1849 //*************************************************************************************************
1850 
1851 
1852 //*************************************************************************************************
1858 template< typename MT // Type of the dense matrix
1859  , size_t... CSAs > // Compile time submatrix arguments
1860 inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::capacity() const noexcept
1861 {
1862  return rows() * columns();
1863 }
1865 //*************************************************************************************************
1866 
1867 
1868 //*************************************************************************************************
1880 template< typename MT // Type of the dense matrix
1881  , size_t... CSAs > // Compile time submatrix arguments
1882 inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::capacity( size_t i ) const noexcept
1883 {
1884  MAYBE_UNUSED( i );
1885 
1886  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1887 
1888  return columns();
1889 }
1891 //*************************************************************************************************
1892 
1893 
1894 //*************************************************************************************************
1900 template< typename MT // Type of the dense matrix
1901  , size_t... CSAs > // Compile time submatrix arguments
1903 {
1904  const size_t iend( row() + rows() );
1905  const size_t jend( column() + columns() );
1906  size_t nonzeros( 0UL );
1907 
1908  for( size_t i=row(); i<iend; ++i )
1909  for( size_t j=column(); j<jend; ++j )
1910  if( !isDefault( matrix_(i,j) ) )
1911  ++nonzeros;
1912 
1913  return nonzeros;
1914 }
1916 //*************************************************************************************************
1917 
1918 
1919 //*************************************************************************************************
1931 template< typename MT // Type of the dense matrix
1932  , size_t... CSAs > // Compile time submatrix arguments
1933 inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::nonZeros( size_t i ) const
1934 {
1935  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1936 
1937  const size_t jend( column() + columns() );
1938  size_t nonzeros( 0UL );
1939 
1940  for( size_t j=column(); j<jend; ++j )
1941  if( !isDefault( matrix_(row()+i,j) ) )
1942  ++nonzeros;
1943 
1944  return nonzeros;
1945 }
1947 //*************************************************************************************************
1948 
1949 
1950 //*************************************************************************************************
1956 template< typename MT // Type of the dense matrix
1957  , size_t... CSAs > // Compile time submatrix arguments
1959 {
1960  using blaze::clear;
1961 
1962  for( size_t i=row(); i<row()+rows(); ++i )
1963  {
1964  const size_t jbegin( ( IsUpper_v<MT> )
1965  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
1966  ?( max( i+1UL, column() ) )
1967  :( max( i, column() ) ) )
1968  :( column() ) );
1969  const size_t jend ( ( IsLower_v<MT> )
1970  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
1971  ?( min( i, column()+columns() ) )
1972  :( min( i+1UL, column()+columns() ) ) )
1973  :( column()+columns() ) );
1974 
1975  for( size_t j=jbegin; j<jend; ++j )
1976  clear( matrix_(i,j) );
1977  }
1978 }
1980 //*************************************************************************************************
1981 
1982 
1983 //*************************************************************************************************
1995 template< typename MT // Type of the dense matrix
1996  , size_t... CSAs > // Compile time submatrix arguments
1998 {
1999  using blaze::clear;
2000 
2001  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
2002 
2003  const size_t jbegin( ( IsUpper_v<MT> )
2004  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
2005  ?( max( i+1UL, column() ) )
2006  :( max( i, column() ) ) )
2007  :( column() ) );
2008  const size_t jend ( ( IsLower_v<MT> )
2009  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
2010  ?( min( i, column()+columns() ) )
2011  :( min( i+1UL, column()+columns() ) ) )
2012  :( column()+columns() ) );
2013 
2014  for( size_t j=jbegin; j<jend; ++j )
2015  clear( matrix_(row()+i,j) );
2016 }
2018 //*************************************************************************************************
2019 
2020 
2021 //*************************************************************************************************
2031 template< typename MT // Type of the dense matrix
2032  , size_t... CSAs > // Compile time submatrix arguments
2033 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::hasOverlap() const noexcept
2034 {
2035  BLAZE_INTERNAL_ASSERT( IsSymmetric_v<MT> || IsHermitian_v<MT>, "Invalid matrix detected" );
2036 
2037  if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
2038  return false;
2039  else return true;
2040 }
2042 //*************************************************************************************************
2043 
2044 
2045 
2046 
2047 //=================================================================================================
2048 //
2049 // NUMERIC FUNCTIONS
2050 //
2051 //=================================================================================================
2052 
2053 //*************************************************************************************************
2071 template< typename MT // Type of the dense matrix
2072  , size_t... CSAs > // Compile time submatrix arguments
2073 inline Submatrix<MT,unaligned,false,true,CSAs...>&
2075 {
2076  if( rows() != columns() ) {
2077  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
2078  }
2079 
2080  if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
2081  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
2082  }
2083 
2084  decltype(auto) left( derestrict( *this ) );
2085  const ResultType tmp( trans( *this ) );
2086 
2087  smpAssign( left, tmp );
2088 
2089  return *this;
2090 }
2092 //*************************************************************************************************
2093 
2094 
2095 //*************************************************************************************************
2113 template< typename MT // Type of the dense matrix
2114  , size_t... CSAs > // Compile time submatrix arguments
2115 inline Submatrix<MT,unaligned,false,true,CSAs...>&
2116  Submatrix<MT,unaligned,false,true,CSAs...>::ctranspose()
2117 {
2118  if( rows() != columns() ) {
2119  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
2120  }
2121 
2122  if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
2123  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
2124  }
2125 
2126  decltype(auto) left( derestrict( *this ) );
2127  const ResultType tmp( ctrans( *this ) );
2128 
2129  smpAssign( left, tmp );
2130 
2131  return *this;
2132 }
2134 //*************************************************************************************************
2135 
2136 
2137 //*************************************************************************************************
2150 template< typename MT // Type of the dense matrix
2151  , size_t... CSAs > // Compile time submatrix arguments
2152 template< typename Other > // Data type of the scalar value
2153 inline Submatrix<MT,unaligned,false,true,CSAs...>&
2154  Submatrix<MT,unaligned,false,true,CSAs...>::scale( const Other& scalar )
2155 {
2157 
2158  const size_t iend( row() + rows() );
2159 
2160  for( size_t i=row(); i<iend; ++i )
2161  {
2162  const size_t jbegin( ( IsUpper_v<MT> )
2163  ?( ( IsStrictlyUpper_v<MT> )
2164  ?( max( i+1UL, column() ) )
2165  :( max( i, column() ) ) )
2166  :( column() ) );
2167  const size_t jend ( ( IsLower_v<MT> )
2168  ?( ( IsStrictlyLower_v<MT> )
2169  ?( min( i, column()+columns() ) )
2170  :( min( i+1UL, column()+columns() ) ) )
2171  :( column()+columns() ) );
2172 
2173  for( size_t j=jbegin; j<jend; ++j )
2174  matrix_(i,j) *= scalar;
2175  }
2176 
2177  return *this;
2178 }
2180 //*************************************************************************************************
2181 
2182 
2183 
2184 
2185 //=================================================================================================
2186 //
2187 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
2188 //
2189 //=================================================================================================
2190 
2191 //*************************************************************************************************
2202 template< typename MT // Type of the dense matrix
2203  , size_t... CSAs > // Compile time submatrix arguments
2204 template< typename Other > // Data type of the foreign expression
2205 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::canAlias( const Other* alias ) const noexcept
2206 {
2207  return matrix_.isAliased( alias );
2208 }
2210 //*************************************************************************************************
2211 
2212 
2213 //*************************************************************************************************
2224 template< typename MT // Type of the dense matrix
2225  , size_t... CSAs > // Compile time submatrix arguments
2226 template< typename MT2 // Data type of the foreign dense submatrix
2227  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
2228  , bool SO2 // Storage order of the foreign dense submatrix
2229  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
2230 inline bool
2231  Submatrix<MT,unaligned,false,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
2232 {
2233  return ( matrix_.isAliased( &alias->matrix_ ) &&
2234  ( row() + rows() > alias->row() ) &&
2235  ( row() < alias->row() + alias->rows() ) &&
2236  ( column() + columns() > alias->column() ) &&
2237  ( column() < alias->column() + alias->columns() ) );
2238 }
2240 //*************************************************************************************************
2241 
2242 
2243 //*************************************************************************************************
2254 template< typename MT // Type of the dense matrix
2255  , size_t... CSAs > // Compile time submatrix arguments
2256 template< typename Other > // Data type of the foreign expression
2257 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::isAliased( const Other* alias ) const noexcept
2258 {
2259  return matrix_.isAliased( alias );
2260 }
2262 //*************************************************************************************************
2263 
2264 
2265 //*************************************************************************************************
2276 template< typename MT // Type of the dense matrix
2277  , size_t... CSAs > // Compile time submatrix arguments
2278 template< typename MT2 // Data type of the foreign dense submatrix
2279  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
2280  , bool SO2 // Storage order of the foreign dense submatrix
2281  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
2282 inline bool
2283  Submatrix<MT,unaligned,false,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
2284 {
2285  return ( matrix_.isAliased( &alias->matrix_ ) &&
2286  ( row() + rows() > alias->row() ) &&
2287  ( row() < alias->row() + alias->rows() ) &&
2288  ( column() + columns() > alias->column() ) &&
2289  ( column() < alias->column() + alias->columns() ) );
2290 }
2292 //*************************************************************************************************
2293 
2294 
2295 //*************************************************************************************************
2305 template< typename MT // Type of the dense matrix
2306  , size_t... CSAs > // Compile time submatrix arguments
2307 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::isAligned() const noexcept
2308 {
2309  return isAligned_;
2310 }
2312 //*************************************************************************************************
2313 
2314 
2315 //*************************************************************************************************
2326 template< typename MT // Type of the dense matrix
2327  , size_t... CSAs > // Compile time submatrix arguments
2328 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::canSMPAssign() const noexcept
2329 {
2330  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
2331 }
2333 //*************************************************************************************************
2334 
2335 
2336 //*************************************************************************************************
2352 template< typename MT // Type of the dense matrix
2353  , size_t... CSAs > // Compile time submatrix arguments
2354 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,false,true,CSAs...>::SIMDType
2355  Submatrix<MT,unaligned,false,true,CSAs...>::load( size_t i, size_t j ) const noexcept
2356 {
2357  if( isAligned_ )
2358  return loada( i, j );
2359  else
2360  return loadu( i, j );
2361 }
2363 //*************************************************************************************************
2364 
2365 
2366 //*************************************************************************************************
2382 template< typename MT // Type of the dense matrix
2383  , size_t... CSAs > // Compile time submatrix arguments
2384 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,false,true,CSAs...>::SIMDType
2385  Submatrix<MT,unaligned,false,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
2386 {
2388 
2389  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2390  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2391  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2392  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2393 
2394  return matrix_.loada( row()+i, column()+j );
2395 }
2397 //*************************************************************************************************
2398 
2399 
2400 //*************************************************************************************************
2416 template< typename MT // Type of the dense matrix
2417  , size_t... CSAs > // Compile time submatrix arguments
2418 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,false,true,CSAs...>::SIMDType
2419  Submatrix<MT,unaligned,false,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
2420 {
2422 
2423  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2424  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2425  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2426 
2427  return matrix_.loadu( row()+i, column()+j );
2428 }
2430 //*************************************************************************************************
2431 
2432 
2433 //*************************************************************************************************
2450 template< typename MT // Type of the dense matrix
2451  , size_t... CSAs > // Compile time submatrix arguments
2453  Submatrix<MT,unaligned,false,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
2454 {
2455  if( isAligned_ )
2456  storea( i, j, value );
2457  else
2458  storeu( i, j, value );
2459 }
2461 //*************************************************************************************************
2462 
2463 
2464 //*************************************************************************************************
2481 template< typename MT // Type of the dense matrix
2482  , size_t... CSAs > // Compile time submatrix arguments
2484  Submatrix<MT,unaligned,false,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
2485 {
2487 
2488  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2489  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2490  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2491  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2492 
2493  matrix_.storea( row()+i, column()+j, value );
2494 }
2496 //*************************************************************************************************
2497 
2498 
2499 //*************************************************************************************************
2516 template< typename MT // Type of the dense matrix
2517  , size_t... CSAs > // Compile time submatrix arguments
2519  Submatrix<MT,unaligned,false,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
2520 {
2522 
2523  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2524  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2525  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2526 
2527  matrix_.storeu( row()+i, column()+j, value );
2528 }
2530 //*************************************************************************************************
2531 
2532 
2533 //*************************************************************************************************
2550 template< typename MT // Type of the dense matrix
2551  , size_t... CSAs > // Compile time submatrix arguments
2553  Submatrix<MT,unaligned,false,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
2554 {
2556 
2557  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2558  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2559  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2560  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2561 
2562  if( isAligned_ )
2563  matrix_.stream( row()+i, column()+j, value );
2564  else
2565  matrix_.storeu( row()+i, column()+j, value );
2566 }
2568 //*************************************************************************************************
2569 
2570 
2571 //*************************************************************************************************
2583 template< typename MT // Type of the dense matrix
2584  , size_t... CSAs > // Compile time submatrix arguments
2585 template< typename MT2 > // Type of the right-hand side dense matrix
2586 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
2587  -> DisableIf_t< VectorizedAssign_v<MT2> >
2588 {
2589  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2590  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2591 
2592  const size_t jpos( columns() & size_t(-2) );
2593  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
2594 
2595  for( size_t i=0UL; i<rows(); ++i ) {
2596  for( size_t j=0UL; j<jpos; j+=2UL ) {
2597  matrix_(row()+i,column()+j ) = (~rhs)(i,j );
2598  matrix_(row()+i,column()+j+1UL) = (~rhs)(i,j+1UL);
2599  }
2600  if( jpos < columns() ) {
2601  matrix_(row()+i,column()+jpos) = (~rhs)(i,jpos);
2602  }
2603  }
2604 }
2606 //*************************************************************************************************
2607 
2608 
2609 //*************************************************************************************************
2621 template< typename MT // Type of the dense matrix
2622  , size_t... CSAs > // Compile time submatrix arguments
2623 template< typename MT2 > // Type of the right-hand side dense matrix
2624 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
2625  -> EnableIf_t< VectorizedAssign_v<MT2> >
2626 {
2628 
2629  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2630  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2631 
2632  const size_t jpos( columns() & size_t(-SIMDSIZE) );
2633  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
2634 
2635  if( useStreaming && isAligned_ &&
2636  rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
2637  !(~rhs).isAliased( &matrix_ ) )
2638  {
2639  for( size_t i=0UL; i<rows(); ++i )
2640  {
2641  size_t j( 0UL );
2642  Iterator left( begin(i) );
2643  ConstIterator_t<MT2> right( (~rhs).begin(i) );
2644 
2645  for( ; j<jpos; j+=SIMDSIZE ) {
2646  left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2647  }
2648  for( ; j<columns(); ++j ) {
2649  *left = *right;
2650  }
2651  }
2652  }
2653  else
2654  {
2655  for( size_t i=0UL; i<rows(); ++i )
2656  {
2657  size_t j( 0UL );
2658  Iterator left( begin(i) );
2659  ConstIterator_t<MT2> right( (~rhs).begin(i) );
2660 
2661  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
2662  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2663  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2664  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2665  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2666  }
2667  for( ; j<jpos; j+=SIMDSIZE ) {
2668  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2669  }
2670  for( ; j<columns(); ++j ) {
2671  *left = *right; ++left; ++right;
2672  }
2673  }
2674  }
2675 }
2677 //*************************************************************************************************
2678 
2679 
2680 //*************************************************************************************************
2692 template< typename MT // Type of the dense matrix
2693  , size_t... CSAs > // Compile time submatrix arguments
2694 template< typename MT2 > // Type of the right-hand side dense matrix
2695 inline void Submatrix<MT,unaligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
2696 {
2698 
2699  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2700  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2701 
2702  constexpr size_t block( BLOCK_SIZE );
2703 
2704  for( size_t ii=0UL; ii<rows(); ii+=block ) {
2705  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
2706  for( size_t jj=0UL; jj<columns(); jj+=block ) {
2707  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
2708  for( size_t i=ii; i<iend; ++i ) {
2709  for( size_t j=jj; j<jend; ++j ) {
2710  matrix_(row()+i,column()+j) = (~rhs)(i,j);
2711  }
2712  }
2713  }
2714  }
2715 }
2717 //*************************************************************************************************
2718 
2719 
2720 //*************************************************************************************************
2732 template< typename MT // Type of the dense matrix
2733  , size_t... CSAs > // Compile time submatrix arguments
2734 template< typename MT2 > // Type of the right-hand side sparse matrix
2735 inline void Submatrix<MT,unaligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
2736 {
2737  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2738  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2739 
2740  for( size_t i=0UL; i<rows(); ++i )
2741  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
2742  matrix_(row()+i,column()+element->index()) = element->value();
2743 }
2745 //*************************************************************************************************
2746 
2747 
2748 //*************************************************************************************************
2760 template< typename MT // Type of the dense matrix
2761  , size_t... CSAs > // Compile time submatrix arguments
2762 template< typename MT2 > // Type of the right-hand side sparse matrix
2763 inline void Submatrix<MT,unaligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
2764 {
2766 
2767  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2768  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2769 
2770  for( size_t j=0UL; j<columns(); ++j )
2771  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
2772  matrix_(row()+element->index(),column()+j) = element->value();
2773 }
2775 //*************************************************************************************************
2776 
2777 
2778 //*************************************************************************************************
2790 template< typename MT // Type of the dense matrix
2791  , size_t... CSAs > // Compile time submatrix arguments
2792 template< typename MT2 > // Type of the right-hand side dense matrix
2793 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
2794  -> DisableIf_t< VectorizedAddAssign_v<MT2> >
2795 {
2796  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2797  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2798 
2799  const size_t jpos( columns() & size_t(-2) );
2800  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
2801 
2802  for( size_t i=0UL; i<rows(); ++i )
2803  {
2804  if( IsDiagonal_v<MT2> ) {
2805  matrix_(row()+i,column()+i) += (~rhs)(i,i);
2806  }
2807  else {
2808  for( size_t j=0UL; j<jpos; j+=2UL ) {
2809  matrix_(row()+i,column()+j ) += (~rhs)(i,j );
2810  matrix_(row()+i,column()+j+1UL) += (~rhs)(i,j+1UL);
2811  }
2812  if( jpos < columns() ) {
2813  matrix_(row()+i,column()+jpos) += (~rhs)(i,jpos);
2814  }
2815  }
2816  }
2817 }
2819 //*************************************************************************************************
2820 
2821 
2822 //*************************************************************************************************
2834 template< typename MT // Type of the dense matrix
2835  , size_t... CSAs > // Compile time submatrix arguments
2836 template< typename MT2 > // Type of the right-hand side dense matrix
2837 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
2838  -> EnableIf_t< VectorizedAddAssign_v<MT2> >
2839 {
2841 
2842  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2843  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2844 
2845  for( size_t i=0UL; i<rows(); ++i )
2846  {
2847  const size_t jbegin( ( IsUpper_v<MT2> )
2848  ?( ( IsStrictlyUpper_v<MT2> ? i+1UL : i ) & size_t(-SIMDSIZE) )
2849  :( 0UL ) );
2850  const size_t jend ( ( IsLower_v<MT2> )
2851  ?( IsStrictlyLower_v<MT2> ? i : i+1UL )
2852  :( columns() ) );
2853  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
2854 
2855  const size_t jpos( jend & size_t(-SIMDSIZE) );
2856  BLAZE_INTERNAL_ASSERT( ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
2857 
2858  size_t j( jbegin );
2859  Iterator left( begin(i) + jbegin );
2860  ConstIterator_t<MT2> right( (~rhs).begin(i) + jbegin );
2861 
2862  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
2863  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2864  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2865  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2866  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2867  }
2868  for( ; j<jpos; j+=SIMDSIZE ) {
2869  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2870  }
2871  for( ; j<jend; ++j ) {
2872  *left += *right; ++left; ++right;
2873  }
2874  }
2875 }
2877 //*************************************************************************************************
2878 
2879 
2880 //*************************************************************************************************
2892 template< typename MT // Type of the dense matrix
2893  , size_t... CSAs > // Compile time submatrix arguments
2894 template< typename MT2 > // Type of the right-hand side dense matrix
2895 inline void Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
2896 {
2898 
2899  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2900  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2901 
2902  constexpr size_t block( BLOCK_SIZE );
2903 
2904  for( size_t ii=0UL; ii<rows(); ii+=block ) {
2905  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
2906  for( size_t jj=0UL; jj<columns(); jj+=block ) {
2907  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
2908  for( size_t i=ii; i<iend; ++i ) {
2909  for( size_t j=jj; j<jend; ++j ) {
2910  matrix_(row()+i,column()+j) += (~rhs)(i,j);
2911  }
2912  }
2913  }
2914  }
2915 }
2917 //*************************************************************************************************
2918 
2919 
2920 //*************************************************************************************************
2932 template< typename MT // Type of the dense matrix
2933  , size_t... CSAs > // Compile time submatrix arguments
2934 template< typename MT2 > // Type of the right-hand side sparse matrix
2935 inline void Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
2936 {
2937  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2938  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2939 
2940  for( size_t i=0UL; i<rows(); ++i )
2941  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
2942  matrix_(row()+i,column()+element->index()) += element->value();
2943 }
2945 //*************************************************************************************************
2946 
2947 
2948 //*************************************************************************************************
2960 template< typename MT // Type of the dense matrix
2961  , size_t... CSAs > // Compile time submatrix arguments
2962 template< typename MT2 > // Type of the right-hand side sparse matrix
2963 inline void Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
2964 {
2966 
2967  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2968  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2969 
2970  for( size_t j=0UL; j<columns(); ++j )
2971  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
2972  matrix_(row()+element->index(),column()+j) += element->value();
2973 }
2975 //*************************************************************************************************
2976 
2977 
2978 //*************************************************************************************************
2990 template< typename MT // Type of the dense matrix
2991  , size_t... CSAs > // Compile time submatrix arguments
2992 template< typename MT2 > // Type of the right-hand side dense matrix
2993 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
2994  -> DisableIf_t< VectorizedSubAssign_v<MT2> >
2995 {
2996  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2997  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2998 
2999  const size_t jpos( columns() & size_t(-2) );
3000  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
3001 
3002  for( size_t i=0UL; i<rows(); ++i )
3003  {
3004  if( IsDiagonal_v<MT2> ) {
3005  matrix_(row()+i,column()+i) -= (~rhs)(i,i);
3006  }
3007  else {
3008  for( size_t j=0UL; j<jpos; j+=2UL ) {
3009  matrix_(row()+i,column()+j ) -= (~rhs)(i,j );
3010  matrix_(row()+i,column()+j+1UL) -= (~rhs)(i,j+1UL);
3011  }
3012  if( jpos < columns() ) {
3013  matrix_(row()+i,column()+jpos) -= (~rhs)(i,jpos);
3014  }
3015  }
3016  }
3017 }
3019 //*************************************************************************************************
3020 
3021 
3022 //*************************************************************************************************
3034 template< typename MT // Type of the dense matrix
3035  , size_t... CSAs > // Compile time submatrix arguments
3036 template< typename MT2 > // Type of the right-hand side dense matrix
3037 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
3038  -> EnableIf_t< VectorizedSubAssign_v<MT2> >
3039 {
3041 
3042  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3043  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3044 
3045  for( size_t i=0UL; i<rows(); ++i )
3046  {
3047  const size_t jbegin( ( IsUpper_v<MT2> )
3048  ?( ( IsStrictlyUpper_v<MT2> ? i+1UL : i ) & size_t(-SIMDSIZE) )
3049  :( 0UL ) );
3050  const size_t jend ( ( IsLower_v<MT2> )
3051  ?( IsStrictlyLower_v<MT2> ? i : i+1UL )
3052  :( columns() ) );
3053  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
3054 
3055  const size_t jpos( jend & size_t(-SIMDSIZE) );
3056  BLAZE_INTERNAL_ASSERT( ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
3057 
3058  size_t j( jbegin );
3059  Iterator left( begin(i) + jbegin );
3060  ConstIterator_t<MT2> right( (~rhs).begin(i) + jbegin );
3061 
3062  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
3063  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3064  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3065  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3066  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3067  }
3068  for( ; j<jpos; j+=SIMDSIZE ) {
3069  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3070  }
3071  for( ; j<jend; ++j ) {
3072  *left -= *right; ++left; ++right;
3073  }
3074  }
3075 }
3077 //*************************************************************************************************
3078 
3079 
3080 //*************************************************************************************************
3092 template< typename MT // Type of the dense matrix
3093  , size_t... CSAs > // Compile time submatrix arguments
3094 template< typename MT2 > // Type of the right-hand side dense matrix
3095 inline void Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
3096 {
3098 
3099  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3100  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3101 
3102  constexpr size_t block( BLOCK_SIZE );
3103 
3104  for( size_t ii=0UL; ii<rows(); ii+=block ) {
3105  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
3106  for( size_t jj=0UL; jj<columns(); jj+=block ) {
3107  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
3108  for( size_t i=ii; i<iend; ++i ) {
3109  for( size_t j=jj; j<jend; ++j ) {
3110  matrix_(row()+i,column()+j) -= (~rhs)(i,j);
3111  }
3112  }
3113  }
3114  }
3115 }
3117 //*************************************************************************************************
3118 
3119 
3120 //*************************************************************************************************
3132 template< typename MT // Type of the dense matrix
3133  , size_t... CSAs > // Compile time submatrix arguments
3134 template< typename MT2 > // Type of the right-hand side sparse matrix
3135 inline void Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
3136 {
3137  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3138  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3139 
3140  for( size_t i=0UL; i<rows(); ++i )
3141  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
3142  matrix_(row()+i,column()+element->index()) -= element->value();
3143 }
3145 //*************************************************************************************************
3146 
3147 
3148 //*************************************************************************************************
3160 template< typename MT // Type of the dense matrix
3161  , size_t... CSAs > // Compile time submatrix arguments
3162 template< typename MT2 > // Type of the right-hand side sparse matrix
3163 inline void Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
3164 {
3166 
3167  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3168  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3169 
3170  for( size_t j=0UL; j<columns(); ++j )
3171  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
3172  matrix_(row()+element->index(),column()+j) -= element->value();
3173 }
3175 //*************************************************************************************************
3176 
3177 
3178 //*************************************************************************************************
3190 template< typename MT // Type of the dense matrix
3191  , size_t... CSAs > // Compile time submatrix arguments
3192 template< typename MT2 > // Type of the right-hand side dense matrix
3193 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
3194  -> DisableIf_t< VectorizedSchurAssign_v<MT2> >
3195 {
3196  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3197  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3198 
3199  const size_t jpos( columns() & size_t(-2) );
3200  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
3201 
3202  for( size_t i=0UL; i<rows(); ++i ) {
3203  for( size_t j=0UL; j<jpos; j+=2UL ) {
3204  matrix_(row()+i,column()+j ) *= (~rhs)(i,j );
3205  matrix_(row()+i,column()+j+1UL) *= (~rhs)(i,j+1UL);
3206  }
3207  if( jpos < columns() ) {
3208  matrix_(row()+i,column()+jpos) *= (~rhs)(i,jpos);
3209  }
3210  }
3211 }
3213 //*************************************************************************************************
3214 
3215 
3216 //*************************************************************************************************
3228 template< typename MT // Type of the dense matrix
3229  , size_t... CSAs > // Compile time submatrix arguments
3230 template< typename MT2 > // Type of the right-hand side dense matrix
3231 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
3232  -> EnableIf_t< VectorizedSchurAssign_v<MT2> >
3233 {
3235 
3236  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3237  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3238 
3239  for( size_t i=0UL; i<rows(); ++i )
3240  {
3241  const size_t jpos( columns() & size_t(-SIMDSIZE) );
3242  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
3243 
3244  size_t j( 0UL );
3245  Iterator left( begin(i) );
3246  ConstIterator_t<MT2> right( (~rhs).begin(i) );
3247 
3248  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
3249  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3250  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3251  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3252  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3253  }
3254  for( ; j<jpos; j+=SIMDSIZE ) {
3255  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3256  }
3257  for( ; j<columns(); ++j ) {
3258  *left *= *right; ++left; ++right;
3259  }
3260  }
3261 }
3263 //*************************************************************************************************
3264 
3265 
3266 //*************************************************************************************************
3278 template< typename MT // Type of the dense matrix
3279  , size_t... CSAs > // Compile time submatrix arguments
3280 template< typename MT2 > // Type of the right-hand side dense matrix
3281 inline void Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
3282 {
3284 
3285  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3286  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3287 
3288  constexpr size_t block( BLOCK_SIZE );
3289 
3290  for( size_t ii=0UL; ii<rows(); ii+=block ) {
3291  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
3292  for( size_t jj=0UL; jj<columns(); jj+=block ) {
3293  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
3294  for( size_t i=ii; i<iend; ++i ) {
3295  for( size_t j=jj; j<jend; ++j ) {
3296  matrix_(row()+i,column()+j) *= (~rhs)(i,j);
3297  }
3298  }
3299  }
3300  }
3301 }
3303 //*************************************************************************************************
3304 
3305 
3306 //*************************************************************************************************
3318 template< typename MT // Type of the dense matrix
3319  , size_t... CSAs > // Compile time submatrix arguments
3320 template< typename MT2 > // Type of the right-hand side sparse matrix
3321 inline void Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
3322 {
3323  using blaze::reset;
3324 
3325  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3326  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3327 
3328  for( size_t i=0UL; i<rows(); ++i )
3329  {
3330  size_t j( 0UL );
3331 
3332  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element ) {
3333  for( ; j<element->index(); ++j )
3334  reset( matrix_(row()+i,column()+j) );
3335  matrix_(row()+i,column()+j) *= element->value();
3336  ++j;
3337  }
3338 
3339  for( ; j<columns(); ++j ) {
3340  reset( matrix_(row()+i,column()+j) );
3341  }
3342  }
3343 }
3345 //*************************************************************************************************
3346 
3347 
3348 //*************************************************************************************************
3360 template< typename MT // Type of the dense matrix
3361  , size_t... CSAs > // Compile time submatrix arguments
3362 template< typename MT2 > // Type of the right-hand side sparse matrix
3363 inline void Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
3364 {
3365  using blaze::reset;
3366 
3368 
3369  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3370  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3371 
3372  for( size_t j=0UL; j<columns(); ++j )
3373  {
3374  size_t i( 0UL );
3375 
3376  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element ) {
3377  for( ; i<element->index(); ++i )
3378  reset( matrix_(row()+i,column()+j) );
3379  matrix_(row()+i,column()+j) *= element->value();
3380  ++i;
3381  }
3382 
3383  for( ; i<rows(); ++i ) {
3384  reset( matrix_(row()+i,column()+j) );
3385  }
3386  }
3387 }
3389 //*************************************************************************************************
3390 
3391 
3392 
3393 
3394 
3395 
3396 
3397 
3398 //=================================================================================================
3399 //
3400 // CLASS TEMPLATE SPECIALIZATION FOR UNALIGNED COLUMN-MAJOR DENSE SUBMATRICES
3401 //
3402 //=================================================================================================
3403 
3404 //*************************************************************************************************
3412 template< typename MT // Type of the dense matrix
3413  , size_t... CSAs > // Compile time submatrix arguments
3414 class Submatrix<MT,unaligned,true,true,CSAs...>
3415  : public View< DenseMatrix< Submatrix<MT,unaligned,true,true,CSAs...>, true > >
3416  , private SubmatrixData<CSAs...>
3417 {
3418  private:
3419  //**Type definitions****************************************************************************
3420  using DataType = SubmatrixData<CSAs...>;
3421  using Operand = If_t< IsExpression_v<MT>, MT, MT& >;
3422  //**********************************************************************************************
3423 
3424  //**********************************************************************************************
3426  template< typename MT1, typename MT2 >
3427  static constexpr bool EnforceEvaluation_v =
3428  ( IsRestricted_v<MT1> && RequiresEvaluation_v<MT2> );
3429  //**********************************************************************************************
3430 
3431  public:
3432  //**Type definitions****************************************************************************
3434  using This = Submatrix<MT,unaligned,true,true,CSAs...>;
3435 
3436  using BaseType = DenseMatrix<This,true>;
3437  using ViewedType = MT;
3438  using ResultType = SubmatrixTrait_t<MT,CSAs...>;
3439  using OppositeType = OppositeType_t<ResultType>;
3440  using TransposeType = TransposeType_t<ResultType>;
3441  using ElementType = ElementType_t<MT>;
3442  using SIMDType = SIMDTrait_t<ElementType>;
3443  using ReturnType = ReturnType_t<MT>;
3444  using CompositeType = const Submatrix&;
3445 
3447  using ConstReference = ConstReference_t<MT>;
3448 
3450  using Reference = If_t< IsConst_v<MT>, ConstReference, Reference_t<MT> >;
3451 
3453  using ConstPointer = ConstPointer_t<MT>;
3454 
3456  using Pointer = If_t< IsConst_v<MT> || !HasMutableDataAccess_v<MT>, ConstPointer, Pointer_t<MT> >;
3457  //**********************************************************************************************
3458 
3459  //**SubmatrixIterator class definition**********************************************************
3462  template< typename IteratorType > // Type of the dense matrix iterator
3463  class SubmatrixIterator
3464  {
3465  public:
3466  //**Type definitions*************************************************************************
3468  using IteratorCategory = typename std::iterator_traits<IteratorType>::iterator_category;
3469 
3471  using ValueType = typename std::iterator_traits<IteratorType>::value_type;
3472 
3474  using PointerType = typename std::iterator_traits<IteratorType>::pointer;
3475 
3477  using ReferenceType = typename std::iterator_traits<IteratorType>::reference;
3478 
3480  using DifferenceType = typename std::iterator_traits<IteratorType>::difference_type;
3481 
3482  // STL iterator requirements
3483  using iterator_category = IteratorCategory;
3484  using value_type = ValueType;
3485  using pointer = PointerType;
3486  using reference = ReferenceType;
3487  using difference_type = DifferenceType;
3488  //*******************************************************************************************
3489 
3490  //**Constructor******************************************************************************
3493  inline SubmatrixIterator()
3494  : iterator_ ( ) // Iterator to the current submatrix element
3495  , isAligned_( false ) // Memory alignment flag
3496  {}
3497  //*******************************************************************************************
3498 
3499  //**Constructor******************************************************************************
3507  inline SubmatrixIterator( IteratorType iterator, bool isMemoryAligned )
3508  : iterator_ ( iterator ) // Iterator to the current submatrix element
3509  , isAligned_( isMemoryAligned ) // Memory alignment flag
3510  {}
3511  //*******************************************************************************************
3512 
3513  //**Constructor******************************************************************************
3518  template< typename IteratorType2 >
3519  inline SubmatrixIterator( const SubmatrixIterator<IteratorType2>& it )
3520  : iterator_ ( it.base() ) // Iterator to the current submatrix element
3521  , isAligned_( it.isAligned() ) // Memory alignment flag
3522  {}
3523  //*******************************************************************************************
3524 
3525  //**Addition assignment operator*************************************************************
3531  inline SubmatrixIterator& operator+=( size_t inc ) {
3532  iterator_ += inc;
3533  return *this;
3534  }
3535  //*******************************************************************************************
3536 
3537  //**Subtraction assignment operator**********************************************************
3543  inline SubmatrixIterator& operator-=( size_t dec ) {
3544  iterator_ -= dec;
3545  return *this;
3546  }
3547  //*******************************************************************************************
3548 
3549  //**Prefix increment operator****************************************************************
3554  inline SubmatrixIterator& operator++() {
3555  ++iterator_;
3556  return *this;
3557  }
3558  //*******************************************************************************************
3559 
3560  //**Postfix increment operator***************************************************************
3565  inline const SubmatrixIterator operator++( int ) {
3566  return SubmatrixIterator( iterator_++, isAligned_ );
3567  }
3568  //*******************************************************************************************
3569 
3570  //**Prefix decrement operator****************************************************************
3575  inline SubmatrixIterator& operator--() {
3576  --iterator_;
3577  return *this;
3578  }
3579  //*******************************************************************************************
3580 
3581  //**Postfix decrement operator***************************************************************
3586  inline const SubmatrixIterator operator--( int ) {
3587  return SubmatrixIterator( iterator_--, isAligned_ );
3588  }
3589  //*******************************************************************************************
3590 
3591  //**Element access operator******************************************************************
3596  inline ReferenceType operator*() const {
3597  return *iterator_;
3598  }
3599  //*******************************************************************************************
3600 
3601  //**Element access operator******************************************************************
3606  inline IteratorType operator->() const {
3607  return iterator_;
3608  }
3609  //*******************************************************************************************
3610 
3611  //**Load function****************************************************************************
3621  inline SIMDType load() const noexcept {
3622  if( isAligned_ )
3623  return loada();
3624  else
3625  return loadu();
3626  }
3627  //*******************************************************************************************
3628 
3629  //**Loada function***************************************************************************
3639  inline SIMDType loada() const noexcept {
3640  return iterator_.loada();
3641  }
3642  //*******************************************************************************************
3643 
3644  //**Loadu function***************************************************************************
3654  inline SIMDType loadu() const noexcept {
3655  return iterator_.loadu();
3656  }
3657  //*******************************************************************************************
3658 
3659  //**Store function***************************************************************************
3670  inline void store( const SIMDType& value ) const {
3671  if( isAligned_ ) {
3672  storea( value );
3673  }
3674  else {
3675  storeu( value );
3676  }
3677  }
3678  //*******************************************************************************************
3679 
3680  //**Storea function**************************************************************************
3691  inline void storea( const SIMDType& value ) const {
3692  iterator_.storea( value );
3693  }
3694  //*******************************************************************************************
3695 
3696  //**Storeu function**************************************************************************
3707  inline void storeu( const SIMDType& value ) const {
3708  iterator_.storeu( value );
3709  }
3710  //*******************************************************************************************
3711 
3712  //**Stream function**************************************************************************
3723  inline void stream( const SIMDType& value ) const {
3724  iterator_.stream( value );
3725  }
3726  //*******************************************************************************************
3727 
3728  //**Equality operator************************************************************************
3734  inline bool operator==( const SubmatrixIterator& rhs ) const {
3735  return iterator_ == rhs.iterator_;
3736  }
3737  //*******************************************************************************************
3738 
3739  //**Inequality operator**********************************************************************
3745  inline bool operator!=( const SubmatrixIterator& rhs ) const {
3746  return iterator_ != rhs.iterator_;
3747  }
3748  //*******************************************************************************************
3749 
3750  //**Less-than operator***********************************************************************
3756  inline bool operator<( const SubmatrixIterator& rhs ) const {
3757  return iterator_ < rhs.iterator_;
3758  }
3759  //*******************************************************************************************
3760 
3761  //**Greater-than operator********************************************************************
3767  inline bool operator>( const SubmatrixIterator& rhs ) const {
3768  return iterator_ > rhs.iterator_;
3769  }
3770  //*******************************************************************************************
3771 
3772  //**Less-or-equal-than operator**************************************************************
3778  inline bool operator<=( const SubmatrixIterator& rhs ) const {
3779  return iterator_ <= rhs.iterator_;
3780  }
3781  //*******************************************************************************************
3782 
3783  //**Greater-or-equal-than operator***********************************************************
3789  inline bool operator>=( const SubmatrixIterator& rhs ) const {
3790  return iterator_ >= rhs.iterator_;
3791  }
3792  //*******************************************************************************************
3793 
3794  //**Subtraction operator*********************************************************************
3800  inline DifferenceType operator-( const SubmatrixIterator& rhs ) const {
3801  return iterator_ - rhs.iterator_;
3802  }
3803  //*******************************************************************************************
3804 
3805  //**Addition operator************************************************************************
3812  friend inline const SubmatrixIterator operator+( const SubmatrixIterator& it, size_t inc ) {
3813  return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
3814  }
3815  //*******************************************************************************************
3816 
3817  //**Addition operator************************************************************************
3824  friend inline const SubmatrixIterator operator+( size_t inc, const SubmatrixIterator& it ) {
3825  return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
3826  }
3827  //*******************************************************************************************
3828 
3829  //**Subtraction operator*********************************************************************
3836  friend inline const SubmatrixIterator operator-( const SubmatrixIterator& it, size_t dec ) {
3837  return SubmatrixIterator( it.iterator_ - dec, it.isAligned_ );
3838  }
3839  //*******************************************************************************************
3840 
3841  //**Base function****************************************************************************
3846  inline IteratorType base() const {
3847  return iterator_;
3848  }
3849  //*******************************************************************************************
3850 
3851  //**IsAligned function***********************************************************************
3856  inline bool isAligned() const noexcept {
3857  return isAligned_;
3858  }
3859  //*******************************************************************************************
3860 
3861  private:
3862  //**Member variables*************************************************************************
3863  IteratorType iterator_;
3864  bool isAligned_;
3865  //*******************************************************************************************
3866  };
3867  //**********************************************************************************************
3868 
3869  //**Type definitions****************************************************************************
3871  using ConstIterator = SubmatrixIterator< ConstIterator_t<MT> >;
3872 
3874  using Iterator = If_t< IsConst_v<MT>, ConstIterator, SubmatrixIterator< Iterator_t<MT> > >;
3875  //**********************************************************************************************
3876 
3877  //**Compilation flags***************************************************************************
3879  static constexpr bool simdEnabled = MT::simdEnabled;
3880 
3882  static constexpr bool smpAssignable = MT::smpAssignable;
3883 
3885  static constexpr bool compileTimeArgs = DataType::compileTimeArgs;
3886  //**********************************************************************************************
3887 
3888  //**Constructors********************************************************************************
3891  template< typename... RSAs >
3892  explicit inline Submatrix( MT& matrix, RSAs... args );
3893 
3894  Submatrix( const Submatrix& ) = default;
3896  //**********************************************************************************************
3897 
3898  //**Destructor**********************************************************************************
3901  ~Submatrix() = default;
3903  //**********************************************************************************************
3904 
3905  //**Data access functions***********************************************************************
3908  inline Reference operator()( size_t i, size_t j );
3909  inline ConstReference operator()( size_t i, size_t j ) const;
3910  inline Reference at( size_t i, size_t j );
3911  inline ConstReference at( size_t i, size_t j ) const;
3912  inline Pointer data () noexcept;
3913  inline ConstPointer data () const noexcept;
3914  inline Pointer data ( size_t j ) noexcept;
3915  inline ConstPointer data ( size_t j ) const noexcept;
3916  inline Iterator begin ( size_t j );
3917  inline ConstIterator begin ( size_t j ) const;
3918  inline ConstIterator cbegin( size_t j ) const;
3919  inline Iterator end ( size_t j );
3920  inline ConstIterator end ( size_t j ) const;
3921  inline ConstIterator cend ( size_t j ) const;
3923  //**********************************************************************************************
3924 
3925  //**Assignment operators************************************************************************
3928  inline Submatrix& operator=( const ElementType& rhs );
3929  inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
3930  inline Submatrix& operator=( const Submatrix& rhs );
3931 
3932  template< typename MT2, bool SO >
3933  inline Submatrix& operator=( const Matrix<MT2,SO>& rhs );
3934 
3935  template< typename MT2, bool SO >
3936  inline auto operator+=( const Matrix<MT2,SO>& rhs )
3937  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3938 
3939  template< typename MT2, bool SO >
3940  inline auto operator+=( const Matrix<MT2,SO>& rhs )
3941  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3942 
3943  template< typename MT2, bool SO >
3944  inline auto operator-=( const Matrix<MT2,SO>& rhs )
3945  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3946 
3947  template< typename MT2, bool SO >
3948  inline auto operator-=( const Matrix<MT2,SO>& rhs )
3949  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3950 
3951  template< typename MT2, bool SO >
3952  inline auto operator%=( const Matrix<MT2,SO>& rhs )
3953  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3954 
3955  template< typename MT2, bool SO >
3956  inline auto operator%=( const Matrix<MT2,SO>& rhs )
3957  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3959  //**********************************************************************************************
3960 
3961  //**Utility functions***************************************************************************
3964  using DataType::row;
3965  using DataType::column;
3966  using DataType::rows;
3967  using DataType::columns;
3968 
3969  inline MT& operand() noexcept;
3970  inline const MT& operand() const noexcept;
3971 
3972  inline size_t spacing() const noexcept;
3973  inline size_t capacity() const noexcept;
3974  inline size_t capacity( size_t i ) const noexcept;
3975  inline size_t nonZeros() const;
3976  inline size_t nonZeros( size_t i ) const;
3977  inline void reset();
3978  inline void reset( size_t i );
3980  //**********************************************************************************************
3981 
3982  //**Numeric functions***************************************************************************
3985  inline Submatrix& transpose();
3986  inline Submatrix& ctranspose();
3987 
3988  template< typename Other > inline Submatrix& scale( const Other& scalar );
3990  //**********************************************************************************************
3991 
3992  private:
3993  //**********************************************************************************************
3995  template< typename MT2 >
3996  static constexpr bool VectorizedAssign_v =
3997  ( useOptimizedKernels &&
3998  simdEnabled && MT2::simdEnabled &&
3999  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > );
4000  //**********************************************************************************************
4001 
4002  //**********************************************************************************************
4004  template< typename MT2 >
4005  static constexpr bool VectorizedAddAssign_v =
4006  ( useOptimizedKernels &&
4007  simdEnabled && MT2::simdEnabled &&
4008  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > &&
4009  HasSIMDAdd_v< ElementType, ElementType_t<MT2> > &&
4010  !IsDiagonal_v<MT2> );
4011  //**********************************************************************************************
4012 
4013  //**********************************************************************************************
4015  template< typename MT2 >
4016  static constexpr bool VectorizedSubAssign_v =
4017  ( useOptimizedKernels &&
4018  simdEnabled && MT2::simdEnabled &&
4019  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > &&
4020  HasSIMDSub_v< ElementType, ElementType_t<MT2> > &&
4021  !IsDiagonal_v<MT2> );
4022  //**********************************************************************************************
4023 
4024  //**********************************************************************************************
4026  template< typename MT2 >
4027  static constexpr bool VectorizedSchurAssign_v =
4028  ( useOptimizedKernels &&
4029  simdEnabled && MT2::simdEnabled &&
4030  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > &&
4031  HasSIMDMult_v< ElementType, ElementType_t<MT2> > );
4032  //**********************************************************************************************
4033 
4034  //**SIMD properties*****************************************************************************
4036  static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
4037  //**********************************************************************************************
4038 
4039  public:
4040  //**Expression template evaluation functions****************************************************
4043  template< typename Other >
4044  inline bool canAlias( const Other* alias ) const noexcept;
4045 
4046  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
4047  inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
4048 
4049  template< typename Other >
4050  inline bool isAliased( const Other* alias ) const noexcept;
4051 
4052  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
4053  inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
4054 
4055  inline bool isAligned () const noexcept;
4056  inline bool canSMPAssign() const noexcept;
4057 
4058  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
4059  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
4060  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
4061 
4062  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
4063  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
4064  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
4065  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
4066 
4067  template< typename MT2 >
4068  inline auto assign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT2> >;
4069 
4070  template< typename MT2 >
4071  inline auto assign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT2> >;
4072 
4073  template< typename MT2 > inline void assign( const DenseMatrix<MT2,false>& rhs );
4074  template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
4075  template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
4076 
4077  template< typename MT2 >
4078  inline auto addAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT2> >;
4079 
4080  template< typename MT2 >
4081  inline auto addAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT2> >;
4082 
4083  template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,false>& rhs );
4084  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
4085  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
4086 
4087  template< typename MT2 >
4088  inline auto subAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT2> >;
4089 
4090  template< typename MT2 >
4091  inline auto subAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT2> >;
4092 
4093  template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,false>& rhs );
4094  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
4095  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
4096 
4097  template< typename MT2 >
4098  inline auto schurAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT2> >;
4099 
4100  template< typename MT2 >
4101  inline auto schurAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT2> >;
4102 
4103  template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,false>& rhs );
4104  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
4105  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
4107  //**********************************************************************************************
4108 
4109  private:
4110  //**Utility functions***************************************************************************
4113  inline bool hasOverlap() const noexcept;
4115  //**********************************************************************************************
4116 
4117  //**Member variables****************************************************************************
4120  Operand matrix_;
4121  const bool isAligned_;
4122 
4129  //**********************************************************************************************
4130 
4131  //**Friend declarations*************************************************************************
4132  template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
4133  //**********************************************************************************************
4134 
4135  //**Compile time checks*************************************************************************
4143  //**********************************************************************************************
4144 };
4146 //*************************************************************************************************
4147 
4148 
4149 
4150 
4151 //=================================================================================================
4152 //
4153 // CONSTRUCTORS
4154 //
4155 //=================================================================================================
4156 
4157 //*************************************************************************************************
4170 template< typename MT // Type of the dense matrix
4171  , size_t... CSAs > // Compile time submatrix arguments
4172 template< typename... RSAs > // Runtime submatrix arguments
4173 inline Submatrix<MT,unaligned,true,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
4174  : DataType ( args... ) // Base class initialization
4175  , matrix_ ( matrix ) // The matrix containing the submatrix
4176  , isAligned_( simdEnabled && IsContiguous_v<MT> &&
4177  matrix.data() != nullptr && checkAlignment( data() ) &&
4178  ( columns() < 2UL || ( matrix.spacing() & size_t(-SIMDSIZE) ) == 0UL ) )
4179 {
4180  if( !Contains_v< TypeList<RSAs...>, Unchecked > ) {
4181  if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
4182  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
4183  }
4184  }
4185  else {
4186  BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
4187  BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
4188  }
4189 }
4191 //*************************************************************************************************
4192 
4193 
4194 
4195 
4196 //=================================================================================================
4197 //
4198 // DATA ACCESS FUNCTIONS
4199 //
4200 //=================================================================================================
4201 
4202 //*************************************************************************************************
4213 template< typename MT // Type of the dense matrix
4214  , size_t... CSAs > // Compile time submatrix arguments
4215 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Reference
4216  Submatrix<MT,unaligned,true,true,CSAs...>::operator()( size_t i, size_t j )
4217 {
4218  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
4219  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
4220 
4221  return matrix_(row()+i,column()+j);
4222 }
4224 //*************************************************************************************************
4225 
4226 
4227 //*************************************************************************************************
4238 template< typename MT // Type of the dense matrix
4239  , size_t... CSAs > // Compile time submatrix arguments
4240 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstReference
4241  Submatrix<MT,unaligned,true,true,CSAs...>::operator()( size_t i, size_t j ) const
4242 {
4243  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
4244  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
4245 
4246  return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
4247 }
4249 //*************************************************************************************************
4250 
4251 
4252 //*************************************************************************************************
4264 template< typename MT // Type of the dense matrix
4265  , size_t... CSAs > // Compile time submatrix arguments
4266 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Reference
4267  Submatrix<MT,unaligned,true,true,CSAs...>::at( size_t i, size_t j )
4268 {
4269  if( i >= rows() ) {
4270  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
4271  }
4272  if( j >= columns() ) {
4273  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
4274  }
4275  return (*this)(i,j);
4276 }
4278 //*************************************************************************************************
4279 
4280 
4281 //*************************************************************************************************
4293 template< typename MT // Type of the dense matrix
4294  , size_t... CSAs > // Compile time submatrix arguments
4295 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstReference
4296  Submatrix<MT,unaligned,true,true,CSAs...>::at( size_t i, size_t j ) const
4297 {
4298  if( i >= rows() ) {
4299  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
4300  }
4301  if( j >= columns() ) {
4302  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
4303  }
4304  return (*this)(i,j);
4305 }
4307 //*************************************************************************************************
4308 
4309 
4310 //*************************************************************************************************
4320 template< typename MT // Type of the dense matrix
4321  , size_t... CSAs > // Compile time submatrix arguments
4322 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Pointer
4324 {
4325  return matrix_.data() + row() + column()*spacing();
4326 }
4328 //*************************************************************************************************
4329 
4330 
4331 //*************************************************************************************************
4341 template< typename MT // Type of the dense matrix
4342  , size_t... CSAs > // Compile time submatrix arguments
4343 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstPointer
4345 {
4346  return matrix_.data() + row() + column()*spacing();
4347 }
4349 //*************************************************************************************************
4350 
4351 
4352 //*************************************************************************************************
4361 template< typename MT // Type of the dense matrix
4362  , size_t... CSAs > // Compile time submatrix arguments
4363 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Pointer
4365 {
4366  return matrix_.data() + row() + (column()+j)*spacing();
4367 }
4369 //*************************************************************************************************
4370 
4371 
4372 //*************************************************************************************************
4381 template< typename MT // Type of the dense matrix
4382  , size_t... CSAs > // Compile time submatrix arguments
4383 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstPointer
4384  Submatrix<MT,unaligned,true,true,CSAs...>::data( size_t j ) const noexcept
4385 {
4386  return matrix_.data() + row() + (column()+j)*spacing();
4387 }
4389 //*************************************************************************************************
4390 
4391 
4392 //*************************************************************************************************
4399 template< typename MT // Type of the dense matrix
4400  , size_t... CSAs > // Compile time submatrix arguments
4401 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Iterator
4403 {
4404  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4405  return Iterator( matrix_.begin( column() + j ) + row(), isAligned_ );
4406 }
4408 //*************************************************************************************************
4409 
4410 
4411 //*************************************************************************************************
4418 template< typename MT // Type of the dense matrix
4419  , size_t... CSAs > // Compile time submatrix arguments
4420 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4422 {
4423  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4424  return ConstIterator( matrix_.cbegin( column() + j ) + row(), isAligned_ );
4425 }
4427 //*************************************************************************************************
4428 
4429 
4430 //*************************************************************************************************
4437 template< typename MT // Type of the dense matrix
4438  , size_t... CSAs > // Compile time submatrix arguments
4439 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4441 {
4442  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4443  return ConstIterator( matrix_.cbegin( column() + j ) + row(), isAligned_ );
4444 }
4446 //*************************************************************************************************
4447 
4448 
4449 //*************************************************************************************************
4456 template< typename MT // Type of the dense matrix
4457  , size_t... CSAs > // Compile time submatrix arguments
4458 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Iterator
4460 {
4461  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4462  return Iterator( matrix_.begin( column() + j ) + row() + rows(), isAligned_ );
4463 }
4465 //*************************************************************************************************
4466 
4467 
4468 //*************************************************************************************************
4475 template< typename MT // Type of the dense matrix
4476  , size_t... CSAs > // Compile time submatrix arguments
4477 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4479 {
4480  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4481  return ConstIterator( matrix_.cbegin( column() + j ) + row() + rows(), isAligned_ );
4482 }
4484 //*************************************************************************************************
4485 
4486 
4487 //*************************************************************************************************
4494 template< typename MT // Type of the dense matrix
4495  , size_t... CSAs > // Compile time submatrix arguments
4496 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4498 {
4499  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4500  return ConstIterator( matrix_.cbegin( column() + j ) + row() + rows(), isAligned_ );
4501 }
4503 //*************************************************************************************************
4504 
4505 
4506 
4507 
4508 //=================================================================================================
4509 //
4510 // ASSIGNMENT OPERATORS
4511 //
4512 //=================================================================================================
4513 
4514 //*************************************************************************************************
4525 template< typename MT // Type of the dense matrix
4526  , size_t... CSAs > // Compile time submatrix arguments
4527 inline Submatrix<MT,unaligned,true,true,CSAs...>&
4528  Submatrix<MT,unaligned,true,true,CSAs...>::operator=( const ElementType& rhs )
4529 {
4530  const size_t jend( column() + columns() );
4531  decltype(auto) left( derestrict( matrix_ ) );
4532 
4533  for( size_t j=column(); j<jend; ++j )
4534  {
4535  const size_t ibegin( ( IsLower_v<MT> )
4536  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
4537  ?( max( j+1UL, row() ) )
4538  :( max( j, row() ) ) )
4539  :( row() ) );
4540  const size_t iend ( ( IsUpper_v<MT> )
4541  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
4542  ?( min( j, row()+rows() ) )
4543  :( min( j+1UL, row()+rows() ) ) )
4544  :( row()+rows() ) );
4545 
4546  for( size_t i=ibegin; i<iend; ++i ) {
4547  if( !IsRestricted_v<MT> || IsTriangular_v<MT> || trySet( matrix_, i, j, rhs ) )
4548  left(i,j) = rhs;
4549  }
4550  }
4551 
4552  return *this;
4553 }
4555 //*************************************************************************************************
4556 
4557 
4558 //*************************************************************************************************
4574 template< typename MT // Type of the dense matrix
4575  , size_t... CSAs > // Compile time submatrix arguments
4576 inline Submatrix<MT,unaligned,true,true,CSAs...>&
4577  Submatrix<MT,unaligned,true,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
4578 {
4579  using blaze::reset;
4580 
4581  if( list.size() != rows() ) {
4582  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
4583  }
4584 
4585  if( IsRestricted_v<MT> ) {
4586  const InitializerMatrix<ElementType> tmp( list, columns() );
4587  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
4588  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4589  }
4590  }
4591 
4592  decltype(auto) left( derestrict( *this ) );
4593  size_t i( 0UL );
4594 
4595  for( const auto& rowList : list ) {
4596  size_t j( 0UL );
4597  for( const auto& element : rowList ) {
4598  left(i,j) = element;
4599  ++j;
4600  }
4601  for( ; j<columns(); ++j ) {
4602  reset( left(i,j) );
4603  }
4604  ++i;
4605  }
4606 
4607  return *this;
4608 }
4610 //*************************************************************************************************
4611 
4612 
4613 //*************************************************************************************************
4628 template< typename MT // Type of the dense matrix
4629  , size_t... CSAs > // Compile time submatrix arguments
4630 inline Submatrix<MT,unaligned,true,true,CSAs...>&
4631  Submatrix<MT,unaligned,true,true,CSAs...>::operator=( const Submatrix& rhs )
4632 {
4635 
4636  if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
4637  return *this;
4638 
4639  if( rows() != rhs.rows() || columns() != rhs.columns() ) {
4640  BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
4641  }
4642 
4643  if( !tryAssign( matrix_, rhs, row(), column() ) ) {
4644  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4645  }
4646 
4647  decltype(auto) left( derestrict( *this ) );
4648 
4649  if( rhs.canAlias( &matrix_ ) ) {
4650  const ResultType tmp( rhs );
4651  smpAssign( left, tmp );
4652  }
4653  else {
4654  smpAssign( left, rhs );
4655  }
4656 
4657  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4658 
4659  return *this;
4660 }
4662 //*************************************************************************************************
4663 
4664 
4665 //*************************************************************************************************
4680 template< typename MT // Type of the dense matrix
4681  , size_t... CSAs > // Compile time submatrix arguments
4682 template< typename MT2 // Type of the right-hand side matrix
4683  , bool SO > // Storage order of the right-hand side matrix
4684 inline Submatrix<MT,unaligned,true,true,CSAs...>&
4685  Submatrix<MT,unaligned,true,true,CSAs...>::operator=( const Matrix<MT2,SO>& rhs )
4686 {
4687  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
4688 
4689  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4690  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4691  }
4692 
4693  using Right = If_t< IsRestricted_v<MT>, CompositeType_t<MT2>, const MT2& >;
4694  Right right( ~rhs );
4695 
4696  if( !tryAssign( matrix_, right, row(), column() ) ) {
4697  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4698  }
4699 
4700  decltype(auto) left( derestrict( *this ) );
4701 
4702  if( IsReference_v<Right> && right.canAlias( &matrix_ ) ) {
4703  const ResultType_t<MT2> tmp( right );
4704  if( IsSparseMatrix_v<MT2> )
4705  reset();
4706  smpAssign( left, tmp );
4707  }
4708  else {
4709  if( IsSparseMatrix_v<MT2> )
4710  reset();
4711  smpAssign( left, right );
4712  }
4713 
4714  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4715 
4716  return *this;
4717 }
4719 //*************************************************************************************************
4720 
4721 
4722 //*************************************************************************************************
4736 template< typename MT // Type of the dense matrix
4737  , size_t... CSAs > // Compile time submatrix arguments
4738 template< typename MT2 // Type of the right-hand side matrix
4739  , bool SO > // Storage order of the right-hand side matrix
4740 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
4741  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4742 {
4745  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
4746 
4747  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
4748 
4751 
4752  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4753  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4754  }
4755 
4756  if( !tryAddAssign( matrix_, ~rhs, row(), column() ) ) {
4757  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4758  }
4759 
4760  decltype(auto) left( derestrict( *this ) );
4761 
4762  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
4763  (~rhs).canAlias( &matrix_ ) ) {
4764  const AddType tmp( *this + (~rhs) );
4765  smpAssign( left, tmp );
4766  }
4767  else {
4768  smpAddAssign( left, ~rhs );
4769  }
4770 
4771  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4772 
4773  return *this;
4774 }
4776 //*************************************************************************************************
4777 
4778 
4779 //*************************************************************************************************
4793 template< typename MT // Type of the dense matrix
4794  , size_t... CSAs > // Compile time submatrix arguments
4795 template< typename MT2 // Type of the right-hand side matrix
4796  , bool SO > // Storage order of the right-hand side matrix
4797 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
4798  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4799 {
4802  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
4803 
4804  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
4805 
4808 
4809  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4810  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4811  }
4812 
4813  const AddType tmp( *this + (~rhs) );
4814 
4815  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
4816  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4817  }
4818 
4819  decltype(auto) left( derestrict( *this ) );
4820 
4821  smpAssign( left, tmp );
4822 
4823  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4824 
4825  return *this;
4826 }
4828 //*************************************************************************************************
4829 
4830 
4831 //*************************************************************************************************
4845 template< typename MT // Type of the dense matrix
4846  , size_t... CSAs > // Compile time submatrix arguments
4847 template< typename MT2 // Type of the right-hand side matrix
4848  , bool SO > // Storage order of the right-hand side matrix
4849 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
4850  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4851 {
4854  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
4855 
4856  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
4857 
4860 
4861  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4862  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4863  }
4864 
4865  if( !trySubAssign( matrix_, ~rhs, row(), column() ) ) {
4866  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4867  }
4868 
4869  decltype(auto) left( derestrict( *this ) );
4870 
4871  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
4872  (~rhs).canAlias( &matrix_ ) ) {
4873  const SubType tmp( *this - (~rhs ) );
4874  smpAssign( left, tmp );
4875  }
4876  else {
4877  smpSubAssign( left, ~rhs );
4878  }
4879 
4880  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4881 
4882  return *this;
4883 }
4885 //*************************************************************************************************
4886 
4887 
4888 //*************************************************************************************************
4902 template< typename MT // Type of the dense matrix
4903  , size_t... CSAs > // Compile time submatrix arguments
4904 template< typename MT2 // Type of the right-hand side matrix
4905  , bool SO > // Storage order of the right-hand side matrix
4906 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
4907  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4908 {
4911  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
4912 
4913  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
4914 
4917 
4918  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4919  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4920  }
4921 
4922  const SubType tmp( *this - (~rhs) );
4923 
4924  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
4925  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4926  }
4927 
4928  decltype(auto) left( derestrict( *this ) );
4929 
4930  smpAssign( left, tmp );
4931 
4932  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4933 
4934  return *this;
4935 }
4937 //*************************************************************************************************
4938 
4939 
4940 //*************************************************************************************************
4954 template< typename MT // Type of the dense matrix
4955  , size_t... CSAs > // Compile time submatrix arguments
4956 template< typename MT2 // Type of the right-hand side matrix
4957  , bool SO > // Storage order of the right-hand side matrix
4958 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
4959  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4960 {
4963  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
4964 
4965  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
4966 
4968 
4969  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4970  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4971  }
4972 
4973  if( !trySchurAssign( matrix_, ~rhs, row(), column() ) ) {
4974  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4975  }
4976 
4977  decltype(auto) left( derestrict( *this ) );
4978 
4979  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
4980  (~rhs).canAlias( &matrix_ ) ) {
4981  const SchurType tmp( *this % (~rhs) );
4982  if( IsSparseMatrix_v<SchurType> )
4983  reset();
4984  smpAssign( left, tmp );
4985  }
4986  else {
4987  smpSchurAssign( left, ~rhs );
4988  }
4989 
4990  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4991 
4992  return *this;
4993 }
4995 //*************************************************************************************************
4996 
4997 
4998 //*************************************************************************************************
5012 template< typename MT // Type of the dense matrix
5013  , size_t... CSAs > // Compile time submatrix arguments
5014 template< typename MT2 // Type of the right-hand side matrix
5015  , bool SO > // Storage order of the right-hand side matrix
5016 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
5017  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
5018 {
5021  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
5022 
5023  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
5024 
5026 
5027  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
5028  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
5029  }
5030 
5031  const SchurType tmp( *this % (~rhs) );
5032 
5033  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
5034  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
5035  }
5036 
5037  decltype(auto) left( derestrict( *this ) );
5038 
5039  if( IsSparseMatrix_v<SchurType> ) {
5040  reset();
5041  }
5042 
5043  smpAssign( left, tmp );
5044 
5045  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
5046 
5047  return *this;
5048 }
5050 //*************************************************************************************************
5051 
5052 
5053 
5054 
5055 //=================================================================================================
5056 //
5057 // UTILITY FUNCTIONS
5058 //
5059 //=================================================================================================
5060 
5061 //*************************************************************************************************
5067 template< typename MT // Type of the dense matrix
5068  , size_t... CSAs > // Compile time submatrix arguments
5069 inline MT& Submatrix<MT,unaligned,true,true,CSAs...>::operand() noexcept
5070 {
5071  return matrix_;
5072 }
5074 //*************************************************************************************************
5075 
5076 
5077 //*************************************************************************************************
5083 template< typename MT // Type of the dense matrix
5084  , size_t... CSAs > // Compile time submatrix arguments
5085 inline const MT& Submatrix<MT,unaligned,true,true,CSAs...>::operand() const noexcept
5086 {
5087  return matrix_;
5088 }
5090 //*************************************************************************************************
5091 
5092 
5093 //*************************************************************************************************
5102 template< typename MT // Type of the dense matrix
5103  , size_t... CSAs > // Compile time submatrix arguments
5104 inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::spacing() const noexcept
5105 {
5106  return matrix_.spacing();
5107 }
5109 //*************************************************************************************************
5110 
5111 
5112 //*************************************************************************************************
5118 template< typename MT // Type of the dense matrix
5119  , size_t... CSAs > // Compile time submatrix arguments
5120 inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::capacity() const noexcept
5121 {
5122  return rows() * columns();
5123 }
5125 //*************************************************************************************************
5126 
5127 
5128 //*************************************************************************************************
5135 template< typename MT // Type of the dense matrix
5136  , size_t... CSAs > // Compile time submatrix arguments
5137 inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::capacity( size_t j ) const noexcept
5138 {
5139  MAYBE_UNUSED( j );
5140 
5141  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
5142 
5143  return rows();
5144 }
5146 //*************************************************************************************************
5147 
5148 
5149 //*************************************************************************************************
5155 template< typename MT // Type of the dense matrix
5156  , size_t... CSAs > // Compile time submatrix arguments
5158 {
5159  const size_t iend( row() + rows() );
5160  const size_t jend( column() + columns() );
5161  size_t nonzeros( 0UL );
5162 
5163  for( size_t j=column(); j<jend; ++j )
5164  for( size_t i=row(); i<iend; ++i )
5165  if( !isDefault( matrix_(i,j) ) )
5166  ++nonzeros;
5167 
5168  return nonzeros;
5169 }
5171 //*************************************************************************************************
5172 
5173 
5174 //*************************************************************************************************
5181 template< typename MT // Type of the dense matrix
5182  , size_t... CSAs > // Compile time submatrix arguments
5183 inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::nonZeros( size_t j ) const
5184 {
5185  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
5186 
5187  const size_t iend( row() + rows() );
5188  size_t nonzeros( 0UL );
5189 
5190  for( size_t i=row(); i<iend; ++i )
5191  if( !isDefault( matrix_(i,column()+j) ) )
5192  ++nonzeros;
5193 
5194  return nonzeros;
5195 }
5197 //*************************************************************************************************
5198 
5199 
5200 //*************************************************************************************************
5206 template< typename MT // Type of the dense matrix
5207  , size_t... CSAs > // Compile time submatrix arguments
5209 {
5210  using blaze::clear;
5211 
5212  for( size_t j=column(); j<column()+columns(); ++j )
5213  {
5214  const size_t ibegin( ( IsLower_v<MT> )
5215  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
5216  ?( max( j+1UL, row() ) )
5217  :( max( j, row() ) ) )
5218  :( row() ) );
5219  const size_t iend ( ( IsUpper_v<MT> )
5220  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
5221  ?( min( j, row()+rows() ) )
5222  :( min( j+1UL, row()+rows() ) ) )
5223  :( row()+rows() ) );
5224 
5225  for( size_t i=ibegin; i<iend; ++i )
5226  clear( matrix_(i,j) );
5227  }
5228 }
5230 //*************************************************************************************************
5231 
5232 
5233 //*************************************************************************************************
5240 template< typename MT // Type of the dense matrix
5241  , size_t... CSAs > // Compile time submatrix arguments
5243 {
5244  using blaze::clear;
5245 
5246  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
5247 
5248  const size_t ibegin( ( IsLower_v<MT> )
5249  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
5250  ?( max( j+1UL, row() ) )
5251  :( max( j, row() ) ) )
5252  :( row() ) );
5253  const size_t iend ( ( IsUpper_v<MT> )
5254  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
5255  ?( min( j, row()+rows() ) )
5256  :( min( j+1UL, row()+rows() ) ) )
5257  :( row()+rows() ) );
5258 
5259  for( size_t i=ibegin; i<iend; ++i )
5260  clear( matrix_(i,column()+j) );
5261 }
5263 //*************************************************************************************************
5264 
5265 
5266 //*************************************************************************************************
5276 template< typename MT // Type of the dense matrix
5277  , size_t... CSAs > // Compile time submatrix arguments
5278 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::hasOverlap() const noexcept
5279 {
5280  BLAZE_INTERNAL_ASSERT( IsSymmetric_v<MT> || IsHermitian_v<MT>, "Invalid matrix detected" );
5281 
5282  if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
5283  return false;
5284  else return true;
5285 }
5287 //*************************************************************************************************
5288 
5289 
5290 
5291 
5292 //=================================================================================================
5293 //
5294 // NUMERIC FUNCTIONS
5295 //
5296 //=================================================================================================
5297 
5298 //*************************************************************************************************
5316 template< typename MT // Type of the dense matrix
5317  , size_t... CSAs > // Compile time submatrix arguments
5318 inline Submatrix<MT,unaligned,true,true,CSAs...>&
5320 {
5321  if( rows() != columns() ) {
5322  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
5323  }
5324 
5325  if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
5326  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
5327  }
5328 
5329  decltype(auto) left( derestrict( *this ) );
5330  const ResultType tmp( trans( *this ) );
5331 
5332  smpAssign( left, tmp );
5333 
5334  return *this;
5335 }
5337 //*************************************************************************************************
5338 
5339 
5340 //*************************************************************************************************
5358 template< typename MT // Type of the dense matrix
5359  , size_t... CSAs > // Compile time submatrix arguments
5360 inline Submatrix<MT,unaligned,true,true,CSAs...>&
5361  Submatrix<MT,unaligned,true,true,CSAs...>::ctranspose()
5362 {
5363  if( rows() != columns() ) {
5364  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
5365  }
5366 
5367  if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
5368  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
5369  }
5370 
5371  decltype(auto) left( derestrict( *this ) );
5372  const ResultType tmp( ctrans( *this ) );
5373 
5374  smpAssign( left, tmp );
5375 
5376  return *this;
5377 }
5379 //*************************************************************************************************
5380 
5381 
5382 //*************************************************************************************************
5395 template< typename MT // Type of the dense matrix
5396  , size_t... CSAs > // Compile time submatrix arguments
5397 template< typename Other > // Data type of the scalar value
5398 inline Submatrix<MT,unaligned,true,true,CSAs...>&
5399  Submatrix<MT,unaligned,true,true,CSAs...>::scale( const Other& scalar )
5400 {
5402 
5403  const size_t jend( column() + columns() );
5404 
5405  for( size_t j=column(); j<jend; ++j )
5406  {
5407  const size_t ibegin( ( IsLower_v<MT> )
5408  ?( ( IsStrictlyLower_v<MT> )
5409  ?( max( j+1UL, row() ) )
5410  :( max( j, row() ) ) )
5411  :( row() ) );
5412  const size_t iend ( ( IsUpper_v<MT> )
5413  ?( ( IsStrictlyUpper_v<MT> )
5414  ?( min( j, row()+rows() ) )
5415  :( min( j+1UL, row()+rows() ) ) )
5416  :( row()+rows() ) );
5417 
5418  for( size_t i=ibegin; i<iend; ++i )
5419  matrix_(i,j) *= scalar;
5420  }
5421 
5422  return *this;
5423 }
5425 //*************************************************************************************************
5426 
5427 
5428 
5429 
5430 //=================================================================================================
5431 //
5432 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
5433 //
5434 //=================================================================================================
5435 
5436 //*************************************************************************************************
5447 template< typename MT // Type of the dense matrix
5448  , size_t... CSAs > // Compile time submatrix arguments
5449 template< typename Other > // Data type of the foreign expression
5450 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::canAlias( const Other* alias ) const noexcept
5451 {
5452  return matrix_.isAliased( alias );
5453 }
5455 //*************************************************************************************************
5456 
5457 
5458 //*************************************************************************************************
5469 template< typename MT // Type of the dense matrix
5470  , size_t... CSAs > // Compile time submatrix arguments
5471 template< typename MT2 // Data type of the foreign dense submatrix
5472  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
5473  , bool SO2 // Storage order of the foreign dense submatrix
5474  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
5475 inline bool
5476  Submatrix<MT,unaligned,true,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
5477 {
5478  return ( matrix_.isAliased( &alias->matrix_ ) &&
5479  ( row() + rows() > alias->row() ) &&
5480  ( row() < alias->row() + alias->rows() ) &&
5481  ( column() + columns() > alias->column() ) &&
5482  ( column() < alias->column() + alias->columns() ) );
5483 }
5485 //*************************************************************************************************
5486 
5487 
5488 //*************************************************************************************************
5499 template< typename MT // Type of the dense matrix
5500  , size_t... CSAs > // Compile time submatrix arguments
5501 template< typename Other > // Data type of the foreign expression
5502 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::isAliased( const Other* alias ) const noexcept
5503 {
5504  return matrix_.isAliased( alias );
5505 }
5507 //*************************************************************************************************
5508 
5509 
5510 //*************************************************************************************************
5521 template< typename MT // Type of the dense matrix
5522  , size_t... CSAs > // Compile time submatrix arguments
5523 template< typename MT2 // Data type of the foreign dense submatrix
5524  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
5525  , bool SO2 // Storage order of the foreign dense submatrix
5526  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
5527 inline bool
5528  Submatrix<MT,unaligned,true,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
5529 {
5530  return ( matrix_.isAliased( &alias->matrix_ ) &&
5531  ( row() + rows() > alias->row() ) &&
5532  ( row() < alias->row() + alias->rows() ) &&
5533  ( column() + columns() > alias->column() ) &&
5534  ( column() < alias->column() + alias->columns() ) );
5535 }
5537 //*************************************************************************************************
5538 
5539 
5540 //*************************************************************************************************
5550 template< typename MT // Type of the dense matrix
5551  , size_t... CSAs > // Compile time submatrix arguments
5552 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::isAligned() const noexcept
5553 {
5554  return isAligned_;
5555 }
5557 //*************************************************************************************************
5558 
5559 
5560 //*************************************************************************************************
5571 template< typename MT // Type of the dense matrix
5572  , size_t... CSAs > // Compile time submatrix arguments
5573 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::canSMPAssign() const noexcept
5574 {
5575  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
5576 }
5578 //*************************************************************************************************
5579 
5580 
5581 //*************************************************************************************************
5596 template< typename MT // Type of the dense matrix
5597  , size_t... CSAs > // Compile time submatrix arguments
5598 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,true,true,CSAs...>::SIMDType
5599  Submatrix<MT,unaligned,true,true,CSAs...>::load( size_t i, size_t j ) const noexcept
5600 {
5601  if( isAligned_ )
5602  return loada( i, j );
5603  else
5604  return loadu( i, j );
5605 }
5607 //*************************************************************************************************
5608 
5609 
5610 //*************************************************************************************************
5625 template< typename MT // Type of the dense matrix
5626  , size_t... CSAs > // Compile time submatrix arguments
5627 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,true,true,CSAs...>::SIMDType
5628  Submatrix<MT,unaligned,true,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
5629 {
5631 
5632  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5633  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5634  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5635  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5636 
5637  return matrix_.loada( row()+i, column()+j );
5638 }
5640 //*************************************************************************************************
5641 
5642 
5643 //*************************************************************************************************
5658 template< typename MT // Type of the dense matrix
5659  , size_t... CSAs > // Compile time submatrix arguments
5660 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,true,true,CSAs...>::SIMDType
5661  Submatrix<MT,unaligned,true,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
5662 {
5664 
5665  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5666  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5667  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5668 
5669  return matrix_.loadu( row()+i, column()+j );
5670 }
5672 //*************************************************************************************************
5673 
5674 
5675 //*************************************************************************************************
5691 template< typename MT // Type of the dense matrix
5692  , size_t... CSAs > // Compile time submatrix arguments
5694  Submatrix<MT,unaligned,true,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
5695 {
5696  if( isAligned_ )
5697  storea( i, j, value );
5698  else
5699  storeu( i, j, value );
5700 }
5702 //*************************************************************************************************
5703 
5704 
5705 //*************************************************************************************************
5721 template< typename MT // Type of the dense matrix
5722  , size_t... CSAs > // Compile time submatrix arguments
5724  Submatrix<MT,unaligned,true,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
5725 {
5727 
5728  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5729  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5730  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5731  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5732 
5733  matrix_.storea( row()+i, column()+j, value );
5734 }
5736 //*************************************************************************************************
5737 
5738 
5739 //*************************************************************************************************
5755 template< typename MT // Type of the dense matrix
5756  , size_t... CSAs > // Compile time submatrix arguments
5758  Submatrix<MT,unaligned,true,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
5759 {
5761 
5762  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5763  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5764  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5765 
5766  matrix_.storeu( row()+i, column()+j, value );
5767 }
5769 //*************************************************************************************************
5770 
5771 
5772 //*************************************************************************************************
5789 template< typename MT // Type of the dense matrix
5790  , size_t... CSAs > // Compile time submatrix arguments
5792  Submatrix<MT,unaligned,true,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
5793 {
5795 
5796  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5797  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5798  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5799  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5800 
5801  if( isAligned_ )
5802  matrix_.stream( row()+i, column()+j, value );
5803  else
5804  matrix_.storeu( row()+i, column()+j, value );
5805 }
5807 //*************************************************************************************************
5808 
5809 
5810 //*************************************************************************************************
5822 template< typename MT // Type of the dense matrix
5823  , size_t... CSAs > // Compile time submatrix arguments
5824 template< typename MT2 > // Type of the right-hand side dense matrix
5825 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
5826  -> DisableIf_t< VectorizedAssign_v<MT2> >
5827 {
5828  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
5829  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
5830 
5831  const size_t ipos( rows() & size_t(-2) );
5832  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
5833 
5834  for( size_t j=0UL; j<columns(); ++j ) {
5835  for( size_t i=0UL; i<ipos; i+=2UL ) {
5836  matrix_(row()+i ,column()+j) = (~rhs)(i ,j);
5837  matrix_(row()+i+1UL,column()+j) = (~rhs)(i+1UL,j);
5838  }
5839  if( ipos < rows() ) {
5840  matrix_(row()+ipos,column()+j) = (~rhs)(ipos,j);
5841  }
5842  }
5843 }
5845 //*************************************************************************************************
5846 
5847 
5848 //*************************************************************************************************
5860 template< typename MT // Type of the dense matrix
5861  , size_t... CSAs > // Compile time submatrix arguments
5862 template< typename MT2 > // Type of the right-hand side dense matrix
5863 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
5864  -> EnableIf_t< VectorizedAssign_v<MT2> >
5865 {
5867 
5868  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
5869  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
5870 
5871  const size_t ipos( rows() & size_t(-SIMDSIZE) );
5872  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
5873 
5874  if( useStreaming && isAligned_ &&
5875  rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
5876  !(~rhs).isAliased( &matrix_ ) )
5877  {
5878  for( size_t j=0UL; j<columns(); ++j )
5879  {
5880  size_t i( 0UL );
5881  Iterator left( begin(j) );
5882  ConstIterator_t<MT2> right( (~rhs).begin(j) );
5883 
5884  for( ; i<ipos; i+=SIMDSIZE ) {
5885  left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5886  }
5887  for( ; i<rows(); ++i ) {
5888  *left = *right; ++left; ++right;
5889  }
5890  }
5891  }
5892  else
5893  {
5894  for( size_t j=0UL; j<columns(); ++j )
5895  {
5896  size_t i( 0UL );
5897  Iterator left( begin(j) );
5898  ConstIterator_t<MT2> right( (~rhs).begin(j) );
5899 
5900  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
5901  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5902  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5903  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5904  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5905  }
5906  for( ; i<ipos; i+=SIMDSIZE ) {
5907  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5908  }
5909  for( ; i<rows(); ++i ) {
5910  *left = *right; ++left; ++right;
5911  }
5912  }
5913  }
5914 }
5916 //*************************************************************************************************
5917 
5918 
5919 //*************************************************************************************************
5931 template< typename MT // Type of the dense matrix
5932  , size_t... CSAs > // Compile time submatrix arguments
5933 template< typename MT2 > // Type of the right-hand side dense matrix
5934 inline void Submatrix<MT,unaligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
5935 {
5937 
5938  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
5939  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
5940 
5941  constexpr size_t block( BLOCK_SIZE );
5942 
5943  for( size_t jj=0UL; jj<columns(); jj+=block ) {
5944  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
5945  for( size_t ii=0UL; ii<rows(); ii+=block ) {
5946  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
5947  for( size_t j=jj; j<jend; ++j ) {
5948  for( size_t i=ii; i<iend; ++i ) {
5949  matrix_(row()+i,column()+j) = (~rhs)(i,j);
5950  }
5951  }
5952  }
5953  }
5954 }
5956 //*************************************************************************************************
5957 
5958 
5959 //*************************************************************************************************
5971 template< typename MT // Type of the dense matrix
5972  , size_t... CSAs > // Compile time submatrix arguments
5973 template< typename MT2 > // Type of the right-hand side sparse matrix
5974 inline void Submatrix<MT,unaligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
5975 {
5976  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
5977  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
5978 
5979  for( size_t j=0UL; j<columns(); ++j )
5980  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
5981  matrix_(row()+element->index(),column()+j) = element->value();
5982 }
5984 //*************************************************************************************************
5985 
5986 
5987 //*************************************************************************************************
5999 template< typename MT // Type of the dense matrix
6000  , size_t... CSAs > // Compile time submatrix arguments
6001 template< typename MT2 > // Type of the right-hand side sparse matrix
6002 inline void Submatrix<MT,unaligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
6003 {
6005 
6006  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6007  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6008 
6009  for( size_t i=0UL; i<rows(); ++i )
6010  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
6011  matrix_(row()+i,column()+element->index()) = element->value();
6012 }
6014 //*************************************************************************************************
6015 
6016 
6017 //*************************************************************************************************
6029 template< typename MT // Type of the dense matrix
6030  , size_t... CSAs > // Compile time submatrix arguments
6031 template< typename MT2 > // Type of the right-hand side dense matrix
6032 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
6033  -> DisableIf_t< VectorizedAddAssign_v<MT2> >
6034 {
6035  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6036  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6037 
6038  const size_t ipos( rows() & size_t(-2) );
6039  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
6040 
6041  for( size_t j=0UL; j<columns(); ++j )
6042  {
6043  if( IsDiagonal_v<MT2> ) {
6044  matrix_(row()+j,column()+j) += (~rhs)(j,j);
6045  }
6046  else {
6047  for( size_t i=0UL; i<ipos; i+=2UL ) {
6048  matrix_(row()+i ,column()+j) += (~rhs)(i ,j);
6049  matrix_(row()+i+1UL,column()+j) += (~rhs)(i+1UL,j);
6050  }
6051  if( ipos < rows() ) {
6052  matrix_(row()+ipos,column()+j) += (~rhs)(ipos,j);
6053  }
6054  }
6055  }
6056 }
6058 //*************************************************************************************************
6059 
6060 
6061 //*************************************************************************************************
6073 template< typename MT // Type of the dense matrix
6074  , size_t... CSAs > // Compile time submatrix arguments
6075 template< typename MT2 > // Type of the right-hand side dense matrix
6076 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
6077  -> EnableIf_t< VectorizedAddAssign_v<MT2> >
6078 {
6080 
6081  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6082  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6083 
6084  for( size_t j=0UL; j<columns(); ++j )
6085  {
6086  const size_t ibegin( ( IsLower_v<MT> )
6087  ?( ( IsStrictlyLower_v<MT> ? j+1UL : j ) & size_t(-SIMDSIZE) )
6088  :( 0UL ) );
6089  const size_t iend ( ( IsUpper_v<MT> )
6090  ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
6091  :( rows() ) );
6092  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
6093 
6094  const size_t ipos( iend & size_t(-SIMDSIZE) );
6095  BLAZE_INTERNAL_ASSERT( ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
6096 
6097  size_t i( ibegin );
6098  Iterator left( begin(j) + ibegin );
6099  ConstIterator_t<MT2> right( (~rhs).begin(j) + ibegin );
6100 
6101  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
6102  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6103  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6104  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6105  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6106  }
6107  for( ; i<ipos; i+=SIMDSIZE ) {
6108  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6109  }
6110  for( ; i<iend; ++i ) {
6111  *left += *right; ++left; ++right;
6112  }
6113  }
6114 }
6116 //*************************************************************************************************
6117 
6118 
6119 //*************************************************************************************************
6131 template< typename MT // Type of the dense matrix
6132  , size_t... CSAs > // Compile time submatrix arguments
6133 template< typename MT2 > // Type of the right-hand side dense matrix
6134 inline void Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
6135 {
6137 
6138  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6139  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6140 
6141  constexpr size_t block( BLOCK_SIZE );
6142 
6143  for( size_t jj=0UL; jj<columns(); jj+=block ) {
6144  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
6145  for( size_t ii=0UL; ii<rows(); ii+=block ) {
6146  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
6147  for( size_t j=jj; j<jend; ++j ) {
6148  for( size_t i=ii; i<iend; ++i ) {
6149  matrix_(row()+i,column()+j) += (~rhs)(i,j);
6150  }
6151  }
6152  }
6153  }
6154 }
6156 //*************************************************************************************************
6157 
6158 
6159 //*************************************************************************************************
6171 template< typename MT // Type of the dense matrix
6172  , size_t... CSAs > // Compile time submatrix arguments
6173 template< typename MT2 > // Type of the right-hand side sparse matrix
6174 inline void Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
6175 {
6176  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6177  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6178 
6179  for( size_t j=0UL; j<columns(); ++j )
6180  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
6181  matrix_(row()+element->index(),column()+j) += element->value();
6182 }
6184 //*************************************************************************************************
6185 
6186 
6187 //*************************************************************************************************
6199 template< typename MT // Type of the dense matrix
6200  , size_t... CSAs > // Compile time submatrix arguments
6201 template< typename MT2 > // Type of the right-hand side sparse matrix
6202 inline void Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
6203 {
6205 
6206  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6207  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6208 
6209  for( size_t i=0UL; i<rows(); ++i )
6210  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
6211  matrix_(row()+i,column()+element->index()) += element->value();
6212 }
6214 //*************************************************************************************************
6215 
6216 
6217 //*************************************************************************************************
6229 template< typename MT // Type of the dense matrix
6230  , size_t... CSAs > // Compile time submatrix arguments
6231 template< typename MT2 > // Type of the right-hand side dense matrix
6232 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
6233  -> DisableIf_t< VectorizedSubAssign_v<MT2> >
6234 {
6235  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6236  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6237 
6238  const size_t ipos( rows() & size_t(-2) );
6239  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
6240 
6241  for( size_t j=0UL; j<columns(); ++j )
6242  {
6243  if( IsDiagonal_v<MT2> ) {
6244  matrix_(row()+j,column()+j) -= (~rhs)(j,j);
6245  }
6246  else {
6247  for( size_t i=0UL; i<ipos; i+=2UL ) {
6248  matrix_(row()+i ,column()+j) -= (~rhs)(i ,j);
6249  matrix_(row()+i+1UL,column()+j) -= (~rhs)(i+1UL,j);
6250  }
6251  if( ipos < rows() ) {
6252  matrix_(row()+ipos,column()+j) -= (~rhs)(ipos,j);
6253  }
6254  }
6255  }
6256 }
6258 //*************************************************************************************************
6259 
6260 
6261 //*************************************************************************************************
6273 template< typename MT // Type of the dense matrix
6274  , size_t... CSAs > // Compile time submatrix arguments
6275 template< typename MT2 > // Type of the right-hand side dense matrix
6276 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
6277  -> EnableIf_t< VectorizedSubAssign_v<MT2> >
6278 {
6280 
6281  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6282  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6283 
6284  for( size_t j=0UL; j<columns(); ++j )
6285  {
6286  const size_t ibegin( ( IsLower_v<MT> )
6287  ?( ( IsStrictlyLower_v<MT> ? j+1UL : j ) & size_t(-SIMDSIZE) )
6288  :( 0UL ) );
6289  const size_t iend ( ( IsUpper_v<MT> )
6290  ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
6291  :( rows() ) );
6292  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
6293 
6294  const size_t ipos( iend & size_t(-SIMDSIZE) );
6295  BLAZE_INTERNAL_ASSERT( ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
6296 
6297  size_t i( ibegin );
6298  Iterator left( begin(j) + ibegin );
6299  ConstIterator_t<MT2> right( (~rhs).begin(j) + ibegin );
6300 
6301  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
6302  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6303  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6304  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6305  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6306  }
6307  for( ; i<ipos; i+=SIMDSIZE ) {
6308  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6309  }
6310  for( ; i<iend; ++i ) {
6311  *left -= *right; ++left; ++right;
6312  }
6313  }
6314 }
6316 //*************************************************************************************************
6317 
6318 
6319 //*************************************************************************************************
6331 template< typename MT // Type of the dense matrix
6332  , size_t... CSAs > // Compile time submatrix arguments
6333 template< typename MT2 > // Type of the right-hand side dense matrix
6334 inline void Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
6335 {
6337 
6338  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6339  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6340 
6341  constexpr size_t block( BLOCK_SIZE );
6342 
6343  for( size_t jj=0UL; jj<columns(); jj+=block ) {
6344  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
6345  for( size_t ii=0UL; ii<rows(); ii+=block ) {
6346  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
6347  for( size_t j=jj; j<jend; ++j ) {
6348  for( size_t i=ii; i<iend; ++i ) {
6349  matrix_(row()+i,column()+j) -= (~rhs)(i,j);
6350  }
6351  }
6352  }
6353  }
6354 }
6356 //*************************************************************************************************
6357 
6358 
6359 //*************************************************************************************************
6371 template< typename MT // Type of the dense matrix
6372  , size_t... CSAs > // Compile time submatrix arguments
6373 template< typename MT2 > // Type of the right-hand side sparse matrix
6374 inline void Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
6375 {
6376  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6377  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6378 
6379  for( size_t j=0UL; j<columns(); ++j )
6380  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
6381  matrix_(row()+element->index(),column()+j) -= element->value();
6382 }
6384 //*************************************************************************************************
6385 
6386 
6387 //*************************************************************************************************
6399 template< typename MT // Type of the dense matrix
6400  , size_t... CSAs > // Compile time submatrix arguments
6401 template< typename MT2 > // Type of the right-hand side sparse matrix
6402 inline void Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
6403 {
6405 
6406  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6407  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6408 
6409  for( size_t i=0UL; i<rows(); ++i )
6410  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
6411  matrix_(row()+i,column()+element->index()) -= element->value();
6412 }
6414 //*************************************************************************************************
6415 
6416 
6417 //*************************************************************************************************
6429 template< typename MT // Type of the dense matrix
6430  , size_t... CSAs > // Compile time submatrix arguments
6431 template< typename MT2 > // Type of the right-hand side dense matrix
6432 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
6433  -> DisableIf_t< VectorizedSchurAssign_v<MT2> >
6434 {
6435  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6436  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6437 
6438  const size_t ipos( rows() & size_t(-2) );
6439  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
6440 
6441  for( size_t j=0UL; j<columns(); ++j ) {
6442  for( size_t i=0UL; i<ipos; i+=2UL ) {
6443  matrix_(row()+i ,column()+j) *= (~rhs)(i ,j);
6444  matrix_(row()+i+1UL,column()+j) *= (~rhs)(i+1UL,j);
6445  }
6446  if( ipos < rows() ) {
6447  matrix_(row()+ipos,column()+j) *= (~rhs)(ipos,j);
6448  }
6449  }
6450 }
6452 //*************************************************************************************************
6453 
6454 
6455 //*************************************************************************************************
6468 template< typename MT // Type of the dense matrix
6469  , size_t... CSAs > // Compile time submatrix arguments
6470 template< typename MT2 > // Type of the right-hand side dense matrix
6471 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
6472  -> EnableIf_t< VectorizedSchurAssign_v<MT2> >
6473 {
6475 
6476  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6477  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6478 
6479  for( size_t j=0UL; j<columns(); ++j )
6480  {
6481  const size_t ipos( rows() & size_t(-SIMDSIZE) );
6482  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
6483 
6484  size_t i( 0UL );
6485  Iterator left( begin(j) );
6486  ConstIterator_t<MT2> right( (~rhs).begin(j) );
6487 
6488  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
6489  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6490  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6491  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6492  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6493  }
6494  for( ; i<ipos; i+=SIMDSIZE ) {
6495  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6496  }
6497  for( ; i<rows(); ++i ) {
6498  *left *= *right; ++left; ++right;
6499  }
6500  }
6501 }
6503 //*************************************************************************************************
6504 
6505 
6506 //*************************************************************************************************
6518 template< typename MT // Type of the dense matrix
6519  , size_t... CSAs > // Compile time submatrix arguments
6520 template< typename MT2 > // Type of the right-hand side dense matrix
6521 inline void Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
6522 {
6524 
6525  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6526  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6527 
6528  constexpr size_t block( BLOCK_SIZE );
6529 
6530  for( size_t jj=0UL; jj<columns(); jj+=block ) {
6531  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
6532  for( size_t ii=0UL; ii<rows(); ii+=block ) {
6533  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
6534  for( size_t j=jj; j<jend; ++j ) {
6535  for( size_t i=ii; i<iend; ++i ) {
6536  matrix_(row()+i,column()+j) *= (~rhs)(i,j);
6537  }
6538  }
6539  }
6540  }
6541 }
6543 //*************************************************************************************************
6544 
6545 
6546 //*************************************************************************************************
6558 template< typename MT // Type of the dense matrix
6559  , size_t... CSAs > // Compile time submatrix arguments
6560 template< typename MT2 > // Type of the right-hand side sparse matrix
6561 inline void Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
6562 {
6563  using blaze::reset;
6564 
6565  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6566  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6567 
6568  for( size_t j=0UL; j<columns(); ++j )
6569  {
6570  size_t i( 0UL );
6571 
6572  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element ) {
6573  for( ; i<element->index(); ++i )
6574  reset( matrix_(row()+i,column()+j) );
6575  matrix_(row()+i,column()+j) *= element->value();
6576  ++i;
6577  }
6578 
6579  for( ; i<rows(); ++i ) {
6580  reset( matrix_(row()+i,column()+j) );
6581  }
6582  }
6583 }
6585 //*************************************************************************************************
6586 
6587 
6588 //*************************************************************************************************
6600 template< typename MT // Type of the dense matrix
6601  , size_t... CSAs > // Compile time submatrix arguments
6602 template< typename MT2 > // Type of the right-hand side sparse matrix
6603 inline void Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
6604 {
6605  using blaze::reset;
6606 
6608 
6609  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6610  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6611 
6612  for( size_t i=0UL; i<rows(); ++i )
6613  {
6614  size_t j( 0UL );
6615 
6616  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element ) {
6617  for( ; j<element->index(); ++j )
6618  reset( matrix_(row()+i,column()+j) );
6619  matrix_(row()+i,column()+j) *= element->value();
6620  ++j;
6621  }
6622 
6623  for( ; j<columns(); ++j ) {
6624  reset( matrix_(row()+i,column()+j) );
6625  }
6626  }
6627 }
6629 //*************************************************************************************************
6630 
6631 
6632 
6633 
6634 
6635 
6636 
6637 
6638 //=================================================================================================
6639 //
6640 // CLASS TEMPLATE SPECIALIZATION FOR ALIGNED ROW-MAJOR DENSE SUBMATRICES
6641 //
6642 //=================================================================================================
6643 
6644 //*************************************************************************************************
6652 template< typename MT // Type of the dense matrix
6653  , size_t... CSAs > // Compile time submatrix arguments
6654 class Submatrix<MT,aligned,false,true,CSAs...>
6655  : public View< DenseMatrix< Submatrix<MT,aligned,false,true,CSAs...>, false > >
6656  , private SubmatrixData<CSAs...>
6657 {
6658  private:
6659  //**Type definitions****************************************************************************
6660  using DataType = SubmatrixData<CSAs...>;
6661  using Operand = If_t< IsExpression_v<MT>, MT, MT& >;
6662  //**********************************************************************************************
6663 
6664  //**********************************************************************************************
6666  template< typename MT1, typename MT2 >
6667  static constexpr bool EnforceEvaluation_v =
6668  ( IsRestricted_v<MT1> && RequiresEvaluation_v<MT2> );
6669  //**********************************************************************************************
6670 
6671  public:
6672  //**Type definitions****************************************************************************
6674  using This = Submatrix<MT,aligned,false,true,CSAs...>;
6675 
6676  using BaseType = DenseMatrix<This,false>;
6677  using ViewedType = MT;
6678  using ResultType = SubmatrixTrait_t<MT,CSAs...>;
6679  using OppositeType = OppositeType_t<ResultType>;
6680  using TransposeType = TransposeType_t<ResultType>;
6681  using ElementType = ElementType_t<MT>;
6682  using SIMDType = SIMDTrait_t<ElementType>;
6683  using ReturnType = ReturnType_t<MT>;
6684  using CompositeType = const Submatrix&;
6685 
6687  using ConstReference = ConstReference_t<MT>;
6688 
6690  using Reference = If_t< IsConst_v<MT>, ConstReference, Reference_t<MT> >;
6691 
6693  using ConstPointer = ConstPointer_t<MT>;
6694 
6696  using Pointer = If_t< IsConst_v<MT> || !HasMutableDataAccess_v<MT>, ConstPointer, Pointer_t<MT> >;
6697 
6699  using ConstIterator = ConstIterator_t<MT>;
6700 
6702  using Iterator = If_t< IsConst_v<MT>, ConstIterator, Iterator_t<MT> >;
6703  //**********************************************************************************************
6704 
6705  //**Compilation flags***************************************************************************
6707  static constexpr bool simdEnabled = MT::simdEnabled;
6708 
6710  static constexpr bool smpAssignable = MT::smpAssignable;
6711 
6713  static constexpr bool compileTimeArgs = DataType::compileTimeArgs;
6714  //**********************************************************************************************
6715 
6716  //**Constructors********************************************************************************
6719  template< typename... RSAs >
6720  explicit inline Submatrix( MT& matrix, RSAs... args );
6721 
6722  Submatrix( const Submatrix& ) = default;
6724  //**********************************************************************************************
6725 
6726  //**Destructor**********************************************************************************
6729  ~Submatrix() = default;
6731  //**********************************************************************************************
6732 
6733  //**Data access functions***********************************************************************
6736  inline Reference operator()( size_t i, size_t j );
6737  inline ConstReference operator()( size_t i, size_t j ) const;
6738  inline Reference at( size_t i, size_t j );
6739  inline ConstReference at( size_t i, size_t j ) const;
6740  inline Pointer data () noexcept;
6741  inline ConstPointer data () const noexcept;
6742  inline Pointer data ( size_t i ) noexcept;
6743  inline ConstPointer data ( size_t i ) const noexcept;
6744  inline Iterator begin ( size_t i );
6745  inline ConstIterator begin ( size_t i ) const;
6746  inline ConstIterator cbegin( size_t i ) const;
6747  inline Iterator end ( size_t i );
6748  inline ConstIterator end ( size_t i ) const;
6749  inline ConstIterator cend ( size_t i ) const;
6751  //**********************************************************************************************
6752 
6753  //**Assignment operators************************************************************************
6756  inline Submatrix& operator=( const ElementType& rhs );
6757  inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
6758  inline Submatrix& operator=( const Submatrix& rhs );
6759 
6760  template< typename MT2, bool SO >
6761  inline Submatrix& operator=( const Matrix<MT2,SO>& rhs );
6762 
6763  template< typename MT2, bool SO >
6764  inline auto operator+=( const Matrix<MT2,SO>& rhs )
6765  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6766 
6767  template< typename MT2, bool SO >
6768  inline auto operator+=( const Matrix<MT2,SO>& rhs )
6769  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6770 
6771  template< typename MT2, bool SO >
6772  inline auto operator-=( const Matrix<MT2,SO>& rhs )
6773  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6774 
6775  template< typename MT2, bool SO >
6776  inline auto operator-=( const Matrix<MT2,SO>& rhs )
6777  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6778 
6779  template< typename MT2, bool SO >
6780  inline auto operator%=( const Matrix<MT2,SO>& rhs )
6781  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6782 
6783  template< typename MT2, bool SO >
6784  inline auto operator%=( const Matrix<MT2,SO>& rhs )
6785  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6787  //**********************************************************************************************
6788 
6789  //**Utility functions***************************************************************************
6792  using DataType::row;
6793  using DataType::column;
6794  using DataType::rows;
6795  using DataType::columns;
6796 
6797  inline MT& operand() noexcept;
6798  inline const MT& operand() const noexcept;
6799 
6800  inline size_t spacing() const noexcept;
6801  inline size_t capacity() const noexcept;
6802  inline size_t capacity( size_t i ) const noexcept;
6803  inline size_t nonZeros() const;
6804  inline size_t nonZeros( size_t i ) const;
6805  inline void reset();
6806  inline void reset( size_t i );
6808  //**********************************************************************************************
6809 
6810  //**Numeric functions***************************************************************************
6813  inline Submatrix& transpose();
6814  inline Submatrix& ctranspose();
6815 
6816  template< typename Other > inline Submatrix& scale( const Other& scalar );
6818  //**********************************************************************************************
6819 
6820  private:
6821  //**********************************************************************************************
6823  template< typename MT2 >
6824  static constexpr bool VectorizedAssign_v =
6825  ( useOptimizedKernels &&
6826  simdEnabled && MT2::simdEnabled &&
6827  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > );
6828  //**********************************************************************************************
6829 
6830  //**********************************************************************************************
6832  template< typename MT2 >
6833  static constexpr bool VectorizedAddAssign_v =
6834  ( useOptimizedKernels &&
6835  simdEnabled && MT2::simdEnabled &&
6836  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > &&
6837  HasSIMDAdd_v< ElementType, ElementType_t<MT2> > &&
6838  !IsDiagonal_v<MT2> );
6839  //**********************************************************************************************
6840 
6841  //**********************************************************************************************
6843  template< typename MT2 >
6844  static constexpr bool VectorizedSubAssign_v =
6845  ( useOptimizedKernels &&
6846  simdEnabled && MT2::simdEnabled &&
6847  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > &&
6848  HasSIMDSub_v< ElementType, ElementType_t<MT2> > &&
6849  !IsDiagonal_v<MT2> );
6850  //**********************************************************************************************
6851 
6852  //**********************************************************************************************
6854  template< typename MT2 >
6855  static constexpr bool VectorizedSchurAssign_v =
6856  ( useOptimizedKernels &&
6857  simdEnabled && MT2::simdEnabled &&
6858  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > &&
6859  HasSIMDMult_v< ElementType, ElementType_t<MT2> > );
6860  //**********************************************************************************************
6861 
6862  //**SIMD properties*****************************************************************************
6864  static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
6865  //**********************************************************************************************
6866 
6867  public:
6868  //**Expression template evaluation functions****************************************************
6871  template< typename Other >
6872  inline bool canAlias( const Other* alias ) const noexcept;
6873 
6874  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
6875  inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
6876 
6877  template< typename Other >
6878  inline bool isAliased( const Other* alias ) const noexcept;
6879 
6880  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
6881  inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
6882 
6883  inline bool isAligned () const noexcept;
6884  inline bool canSMPAssign() const noexcept;
6885 
6886  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
6887  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
6888  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
6889 
6890  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
6891  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
6892  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
6893  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
6894 
6895  template< typename MT2 >
6896  inline auto assign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT2> >;
6897 
6898  template< typename MT2 >
6899  inline auto assign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT2> >;
6900 
6901  template< typename MT2 > inline void assign( const DenseMatrix<MT2,true>& rhs );
6902  template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
6903  template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
6904 
6905  template< typename MT2 >
6906  inline auto addAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT2> >;
6907 
6908  template< typename MT2 >
6909  inline auto addAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT2> >;
6910 
6911  template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,true>& rhs );
6912  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
6913  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
6914 
6915  template< typename MT2 >
6916  inline auto subAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT2> >;
6917 
6918  template< typename MT2 >
6919  inline auto subAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT2> >;
6920 
6921  template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,true>& rhs );
6922  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
6923  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
6924 
6925  template< typename MT2 >
6926  inline auto schurAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT2> >;
6927 
6928  template< typename MT2 >
6929  inline auto schurAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT2> >;
6930 
6931  template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,true>& rhs );
6932  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
6933  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
6935  //**********************************************************************************************
6936 
6937  private:
6938  //**Utility functions***************************************************************************
6941  inline bool hasOverlap() const noexcept;
6943  //**********************************************************************************************
6944 
6945  //**Member variables****************************************************************************
6948  Operand matrix_;
6949 
6950  //**********************************************************************************************
6951 
6952  //**Friend declarations*************************************************************************
6953  template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
6954  //**********************************************************************************************
6955 
6956  //**Compile time checks*************************************************************************
6964  //**********************************************************************************************
6965 };
6967 //*************************************************************************************************
6968 
6969 
6970 
6971 
6972 //=================================================================================================
6973 //
6974 // CONSTRUCTORS
6975 //
6976 //=================================================================================================
6977 
6978 //*************************************************************************************************
6991 template< typename MT // Type of the dense matrix
6992  , size_t... CSAs > // Compile time submatrix arguments
6993 template< typename... RSAs > // Runtime submatrix arguments
6994 inline Submatrix<MT,aligned,false,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
6995  : DataType( args... ) // Base class initialization
6996  , matrix_ ( matrix ) // The matrix containing the submatrix
6997 {
6998  if( !Contains_v< TypeList<RSAs...>, Unchecked > )
6999  {
7000  if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
7001  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
7002  }
7003 
7004  if( simdEnabled && IsContiguous_v<MT> &&
7005  ( !checkAlignment( data() ) ||
7006  ( rows() > 1UL && matrix_.spacing() % SIMDSIZE != 0UL ) ) ) {
7007  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix alignment" );
7008  }
7009  }
7010  else
7011  {
7012  BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
7013  BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
7014 
7015  BLAZE_USER_ASSERT( !simdEnabled || !IsContiguous_v<MT> || checkAlignment( data() ), "Invalid submatrix alignment" );
7016  BLAZE_USER_ASSERT( !simdEnabled || !IsContiguous_v<MT> || rows() <= 1UL || matrix_.spacing() % SIMDSIZE == 0UL, "Invalid submatrix alignment" );
7017  }
7018 }
7020 //*************************************************************************************************
7021 
7022 
7023 
7024 
7025 //=================================================================================================
7026 //
7027 // DATA ACCESS FUNCTIONS
7028 //
7029 //=================================================================================================
7030 
7031 //*************************************************************************************************
7042 template< typename MT // Type of the dense matrix
7043  , size_t... CSAs > // Compile time submatrix arguments
7044 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Reference
7045  Submatrix<MT,aligned,false,true,CSAs...>::operator()( size_t i, size_t j )
7046 {
7047  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
7048  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
7049 
7050  return matrix_(row()+i,column()+j);
7051 }
7053 //*************************************************************************************************
7054 
7055 
7056 //*************************************************************************************************
7067 template< typename MT // Type of the dense matrix
7068  , size_t... CSAs > // Compile time submatrix arguments
7069 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstReference
7070  Submatrix<MT,aligned,false,true,CSAs...>::operator()( size_t i, size_t j ) const
7071 {
7072  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
7073  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
7074 
7075  return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
7076 }
7078 //*************************************************************************************************
7079 
7080 
7081 //*************************************************************************************************
7093 template< typename MT // Type of the dense matrix
7094  , size_t... CSAs > // Compile time submatrix arguments
7095 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Reference
7096  Submatrix<MT,aligned,false,true,CSAs...>::at( size_t i, size_t j )
7097 {
7098  if( i >= rows() ) {
7099  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
7100  }
7101  if( j >= columns() ) {
7102  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
7103  }
7104  return (*this)(i,j);
7105 }
7107 //*************************************************************************************************
7108 
7109 
7110 //*************************************************************************************************
7122 template< typename MT // Type of the dense matrix
7123  , size_t... CSAs > // Compile time submatrix arguments
7124 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstReference
7125  Submatrix<MT,aligned,false,true,CSAs...>::at( size_t i, size_t j ) const
7126 {
7127  if( i >= rows() ) {
7128  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
7129  }
7130  if( j >= columns() ) {
7131  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
7132  }
7133  return (*this)(i,j);
7134 }
7136 //*************************************************************************************************
7137 
7138 
7139 //*************************************************************************************************
7149 template< typename MT // Type of the dense matrix
7150  , size_t... CSAs > // Compile time submatrix arguments
7151 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Pointer
7153 {
7154  return matrix_.data() + row()*spacing() + column();
7155 }
7157 //*************************************************************************************************
7158 
7159 
7160 //*************************************************************************************************
7170 template< typename MT // Type of the dense matrix
7171  , size_t... CSAs > // Compile time submatrix arguments
7172 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstPointer
7174 {
7175  return matrix_.data() + row()*spacing() + column();
7176 }
7178 //*************************************************************************************************
7179 
7180 
7181 //*************************************************************************************************
7190 template< typename MT // Type of the dense matrix
7191  , size_t... CSAs > // Compile time submatrix arguments
7192 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Pointer
7194 {
7195  return matrix_.data() + (row()+i)*spacing() + column();
7196 }
7198 //*************************************************************************************************
7199 
7200 
7201 //*************************************************************************************************
7210 template< typename MT // Type of the dense matrix
7211  , size_t... CSAs > // Compile time submatrix arguments
7212 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstPointer
7213  Submatrix<MT,aligned,false,true,CSAs...>::data( size_t i ) const noexcept
7214 {
7215  return matrix_.data() + (row()+i)*spacing() + column();
7216 }
7218 //*************************************************************************************************
7219 
7220 
7221 //*************************************************************************************************
7233 template< typename MT // Type of the dense matrix
7234  , size_t... CSAs > // Compile time submatrix arguments
7235 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Iterator
7237 {
7238  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7239  return ( matrix_.begin( row() + i ) + column() );
7240 }
7242 //*************************************************************************************************
7243 
7244 
7245 //*************************************************************************************************
7257 template< typename MT // Type of the dense matrix
7258  , size_t... CSAs > // Compile time submatrix arguments
7259 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7261 {
7262  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7263  return ( matrix_.cbegin( row() + i ) + column() );
7264 }
7266 //*************************************************************************************************
7267 
7268 
7269 //*************************************************************************************************
7281 template< typename MT // Type of the dense matrix
7282  , size_t... CSAs > // Compile time submatrix arguments
7283 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7285 {
7286  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7287  return ( matrix_.cbegin( row() + i ) + column() );
7288 }
7290 //*************************************************************************************************
7291 
7292 
7293 //*************************************************************************************************
7305 template< typename MT // Type of the dense matrix
7306  , size_t... CSAs > // Compile time submatrix arguments
7307 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Iterator
7309 {
7310  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7311  return ( matrix_.begin( row() + i ) + column() + columns() );
7312 }
7314 //*************************************************************************************************
7315 
7316 
7317 //*************************************************************************************************
7329 template< typename MT // Type of the dense matrix
7330  , size_t... CSAs > // Compile time submatrix arguments
7331 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7333 {
7334  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7335  return ( matrix_.cbegin( row() + i ) + column() + columns() );
7336 }
7338 //*************************************************************************************************
7339 
7340 
7341 //*************************************************************************************************
7353 template< typename MT // Type of the dense matrix
7354  , size_t... CSAs > // Compile time submatrix arguments
7355 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7357 {
7358  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7359  return ( matrix_.cbegin( row() + i ) + column() + columns() );
7360 }
7362 //*************************************************************************************************
7363 
7364 
7365 
7366 
7367 //=================================================================================================
7368 //
7369 // ASSIGNMENT OPERATORS
7370 //
7371 //=================================================================================================
7372 
7373 //*************************************************************************************************
7384 template< typename MT // Type of the dense matrix
7385  , size_t... CSAs > // Compile time submatrix arguments
7386 inline Submatrix<MT,aligned,false,true,CSAs...>&
7387  Submatrix<MT,aligned,false,true,CSAs...>::operator=( const ElementType& rhs )
7388 {
7389  const size_t iend( row() + rows() );
7390  decltype(auto) left( derestrict( matrix_ ) );
7391 
7392  for( size_t i=row(); i<iend; ++i )
7393  {
7394  const size_t jbegin( ( IsUpper_v<MT> )
7395  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
7396  ?( max( i+1UL, column() ) )
7397  :( max( i, column() ) ) )
7398  :( column() ) );
7399  const size_t jend ( ( IsLower_v<MT> )
7400  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
7401  ?( min( i, column()+columns() ) )
7402  :( min( i+1UL, column()+columns() ) ) )
7403  :( column()+columns() ) );
7404 
7405  for( size_t j=jbegin; j<jend; ++j ) {
7406  if( !IsRestricted_v<MT> || IsTriangular_v<MT> || trySet( matrix_, i, j, rhs ) )
7407  left(i,j) = rhs;
7408  }
7409  }
7410 
7411  return *this;
7412 }
7414 //*************************************************************************************************
7415 
7416 
7417 //*************************************************************************************************
7433 template< typename MT // Type of the dense matrix
7434  , size_t... CSAs > // Compile time submatrix arguments
7435 inline Submatrix<MT,aligned,false,true,CSAs...>&
7436  Submatrix<MT,aligned,false,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
7437 {
7438  if( list.size() != rows() ) {
7439  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
7440  }
7441 
7442  if( IsRestricted_v<MT> ) {
7443  const InitializerMatrix<ElementType> tmp( list, columns() );
7444  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7445  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7446  }
7447  }
7448 
7449  decltype(auto) left( derestrict( *this ) );
7450  size_t i( 0UL );
7451 
7452  for( const auto& rowList : list ) {
7453  std::fill( std::copy( rowList.begin(), rowList.end(), left.begin(i) ), left.end(i), ElementType() );
7454  ++i;
7455  }
7456 
7457  return *this;
7458 }
7460 //*************************************************************************************************
7461 
7462 
7463 //*************************************************************************************************
7478 template< typename MT // Type of the dense matrix
7479  , size_t... CSAs > // Compile time submatrix arguments
7480 inline Submatrix<MT,aligned,false,true,CSAs...>&
7481  Submatrix<MT,aligned,false,true,CSAs...>::operator=( const Submatrix& rhs )
7482 {
7485 
7486  if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
7487  return *this;
7488 
7489  if( rows() != rhs.rows() || columns() != rhs.columns() ) {
7490  BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
7491  }
7492 
7493  if( !tryAssign( matrix_, rhs, row(), column() ) ) {
7494  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7495  }
7496 
7497  decltype(auto) left( derestrict( *this ) );
7498 
7499  if( rhs.canAlias( &matrix_ ) ) {
7500  const ResultType tmp( rhs );
7501  smpAssign( left, tmp );
7502  }
7503  else {
7504  smpAssign( left, rhs );
7505  }
7506 
7507  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7508 
7509  return *this;
7510 }
7512 //*************************************************************************************************
7513 
7514 
7515 //*************************************************************************************************
7530 template< typename MT // Type of the dense matrix
7531  , size_t... CSAs > // Compile time submatrix arguments
7532 template< typename MT2 // Type of the right-hand side matrix
7533  , bool SO > // Storage order of the right-hand side matrix
7534 inline Submatrix<MT,aligned,false,true,CSAs...>&
7535  Submatrix<MT,aligned,false,true,CSAs...>::operator=( const Matrix<MT2,SO>& rhs )
7536 {
7537  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7538 
7539  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7540  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7541  }
7542 
7543  using Right = If_t< IsRestricted_v<MT>, CompositeType_t<MT2>, const MT2& >;
7544  Right right( ~rhs );
7545 
7546  if( !tryAssign( matrix_, right, row(), column() ) ) {
7547  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7548  }
7549 
7550  decltype(auto) left( derestrict( *this ) );
7551 
7552  if( IsReference_v<Right> && right.canAlias( &matrix_ ) ) {
7553  const ResultType_t<MT2> tmp( right );
7554  if( IsSparseMatrix_v<MT2> )
7555  reset();
7556  smpAssign( left, tmp );
7557  }
7558  else {
7559  if( IsSparseMatrix_v<MT2> )
7560  reset();
7561  smpAssign( left, right );
7562  }
7563 
7564  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7565 
7566  return *this;
7567 }
7569 //*************************************************************************************************
7570 
7571 
7572 //*************************************************************************************************
7586 template< typename MT // Type of the dense matrix
7587  , size_t... CSAs > // Compile time submatrix arguments
7588 template< typename MT2 // Type of the right-hand side matrix
7589  , bool SO > // Storage order of the right-hand side matrix
7590 inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
7591  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7592 {
7595  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7596 
7597  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
7598 
7601 
7602  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7603  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7604  }
7605 
7606  if( !tryAddAssign( matrix_, ~rhs, row(), column() ) ) {
7607  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7608  }
7609 
7610  decltype(auto) left( derestrict( *this ) );
7611 
7612  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
7613  (~rhs).canAlias( &matrix_ ) ) {
7614  const AddType tmp( *this + (~rhs) );
7615  smpAssign( left, tmp );
7616  }
7617  else {
7618  smpAddAssign( left, ~rhs );
7619  }
7620 
7621  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7622 
7623  return *this;
7624 }
7626 //*************************************************************************************************
7627 
7628 
7629 //*************************************************************************************************
7643 template< typename MT // Type of the dense matrix
7644  , size_t... CSAs > // Compile time submatrix arguments
7645 template< typename MT2 // Type of the right-hand side matrix
7646  , bool SO > // Storage order of the right-hand side matrix
7647 inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
7648  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7649 {
7652  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7653 
7654  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
7655 
7658 
7659  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7660  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7661  }
7662 
7663  const AddType tmp( *this + (~rhs) );
7664 
7665  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7666  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7667  }
7668 
7669  decltype(auto) left( derestrict( *this ) );
7670 
7671  smpAssign( left, tmp );
7672 
7673  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7674 
7675  return *this;
7676 }
7678 //*************************************************************************************************
7679 
7680 
7681 //*************************************************************************************************
7695 template< typename MT // Type of the dense matrix
7696  , size_t... CSAs > // Compile time submatrix arguments
7697 template< typename MT2 // Type of the right-hand side matrix
7698  , bool SO > // Storage order of the right-hand side matrix
7699 inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
7700  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7701 {
7704  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7705 
7706  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
7707 
7710 
7711  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7712  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7713  }
7714 
7715  if( !trySubAssign( matrix_, ~rhs, row(), column() ) ) {
7716  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7717  }
7718 
7719  decltype(auto) left( derestrict( *this ) );
7720 
7721  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
7722  (~rhs).canAlias( &matrix_ ) ) {
7723  const SubType tmp( *this - (~rhs ) );
7724  smpAssign( left, tmp );
7725  }
7726  else {
7727  smpSubAssign( left, ~rhs );
7728  }
7729 
7730  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7731 
7732  return *this;
7733 }
7735 //*************************************************************************************************
7736 
7737 
7738 //*************************************************************************************************
7752 template< typename MT // Type of the dense matrix
7753  , size_t... CSAs > // Compile time submatrix arguments
7754 template< typename MT2 // Type of the right-hand side matrix
7755  , bool SO > // Storage order of the right-hand side matrix
7756 inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
7757  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7758 {
7761  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7762 
7763  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
7764 
7767 
7768  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7769  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7770  }
7771 
7772  const SubType tmp( *this - (~rhs) );
7773 
7774  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7775  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7776  }
7777 
7778  decltype(auto) left( derestrict( *this ) );
7779 
7780  smpAssign( left, tmp );
7781 
7782  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7783 
7784  return *this;
7785 }
7787 //*************************************************************************************************
7788 
7789 
7790 //*************************************************************************************************
7804 template< typename MT // Type of the dense matrix
7805  , size_t... CSAs > // Compile time submatrix arguments
7806 template< typename MT2 // Type of the right-hand side matrix
7807  , bool SO > // Storage order of the right-hand side matrix
7808 inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
7809  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7810 {
7813  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7814 
7815  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
7816 
7818 
7819  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7820  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7821  }
7822 
7823  if( !trySchurAssign( matrix_, ~rhs, row(), column() ) ) {
7824  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7825  }
7826 
7827  decltype(auto) left( derestrict( *this ) );
7828 
7829  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
7830  (~rhs).canAlias( &matrix_ ) ) {
7831  const SchurType tmp( *this % (~rhs) );
7832  if( IsSparseMatrix_v<SchurType> )
7833  reset();
7834  smpAssign( left, tmp );
7835  }
7836  else {
7837  smpSchurAssign( left, ~rhs );
7838  }
7839 
7840  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7841 
7842  return *this;
7843 }
7845 //*************************************************************************************************
7846 
7847 
7848 //*************************************************************************************************
7862 template< typename MT // Type of the dense matrix
7863  , size_t... CSAs > // Compile time submatrix arguments
7864 template< typename MT2 // Type of the right-hand side matrix
7865  , bool SO > // Storage order of the right-hand side matrix
7866 inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
7867  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7868 {
7871  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7872 
7873  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
7874 
7876 
7877  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7878  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7879  }
7880 
7881  const SchurType tmp( *this % (~rhs) );
7882 
7883  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7884  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7885  }
7886 
7887  decltype(auto) left( derestrict( *this ) );
7888 
7889  if( IsSparseMatrix_v<SchurType> ) {
7890  reset();
7891  }
7892 
7893  smpAssign( left, tmp );
7894 
7895  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7896 
7897  return *this;
7898 }
7900 //*************************************************************************************************
7901 
7902 
7903 
7904 
7905 //=================================================================================================
7906 //
7907 // UTILITY FUNCTIONS
7908 //
7909 //=================================================================================================
7910 
7911 //*************************************************************************************************
7917 template< typename MT // Type of the dense matrix
7918  , size_t... CSAs > // Compile time submatrix arguments
7919 inline MT& Submatrix<MT,aligned,false,true,CSAs...>::operand() noexcept
7920 {
7921  return matrix_;
7922 }
7924 //*************************************************************************************************
7925 
7926 
7927 //*************************************************************************************************
7933 template< typename MT // Type of the dense matrix
7934  , size_t... CSAs > // Compile time submatrix arguments
7935 inline const MT& Submatrix<MT,aligned,false,true,CSAs...>::operand() const noexcept
7936 {
7937  return matrix_;
7938 }
7940 //*************************************************************************************************
7941 
7942 
7943 //*************************************************************************************************
7954 template< typename MT // Type of the dense matrix
7955  , size_t... CSAs > // Compile time submatrix arguments
7956 inline size_t Submatrix<MT,aligned,false,true,CSAs...>::spacing() const noexcept
7957 {
7958  return matrix_.spacing();
7959 }
7961 //*************************************************************************************************
7962 
7963 
7964 //*************************************************************************************************
7970 template< typename MT // Type of the dense matrix
7971  , size_t... CSAs > // Compile time submatrix arguments
7972 inline size_t Submatrix<MT,aligned,false,true,CSAs...>::capacity() const noexcept
7973 {
7974  return rows() * columns();
7975 }
7977 //*************************************************************************************************
7978 
7979 
7980 //*************************************************************************************************
7992 template< typename MT // Type of the dense matrix
7993  , size_t... CSAs > // Compile time submatrix arguments
7994 inline size_t Submatrix<MT,aligned,false,true,CSAs...>::capacity( size_t i ) const noexcept
7995 {
7996  MAYBE_UNUSED( i );
7997 
7998  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
7999 
8000  return columns();
8001 }
8003 //*************************************************************************************************
8004 
8005 
8006 //*************************************************************************************************
8012 template< typename MT // Type of the dense matrix
8013  , size_t... CSAs > // Compile time submatrix arguments
8015 {
8016  const size_t iend( row() + rows() );
8017  const size_t jend( column() + columns() );
8018  size_t nonzeros( 0UL );
8019 
8020  for( size_t i=row(); i<iend; ++i )
8021  for( size_t j=column(); j<jend; ++j )
8022  if( !isDefault( matrix_(i,j) ) )
8023  ++nonzeros;
8024 
8025  return nonzeros;
8026 }
8028 //*************************************************************************************************
8029 
8030 
8031 //*************************************************************************************************
8043 template< typename MT // Type of the dense matrix
8044  , size_t... CSAs > // Compile time submatrix arguments
8045 inline size_t Submatrix<MT,aligned,false,true,CSAs...>::nonZeros( size_t i ) const
8046 {
8047  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
8048 
8049  const size_t jend( column() + columns() );
8050  size_t nonzeros( 0UL );
8051 
8052  for( size_t j=column(); j<jend; ++j )
8053  if( !isDefault( matrix_(row()+i,j) ) )
8054  ++nonzeros;
8055 
8056  return nonzeros;
8057 }
8059 //*************************************************************************************************
8060 
8061 
8062 //*************************************************************************************************
8068 template< typename MT // Type of the dense matrix
8069  , size_t... CSAs > // Compile time submatrix arguments
8071 {
8072  using blaze::clear;
8073 
8074  for( size_t i=row(); i<row()+rows(); ++i )
8075  {
8076  const size_t jbegin( ( IsUpper_v<MT> )
8077  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
8078  ?( max( i+1UL, column() ) )
8079  :( max( i, column() ) ) )
8080  :( column() ) );
8081  const size_t jend ( ( IsLower_v<MT> )
8082  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
8083  ?( min( i, column()+columns() ) )
8084  :( min( i+1UL, column()+columns() ) ) )
8085  :( column()+columns() ) );
8086 
8087  for( size_t j=jbegin; j<jend; ++j )
8088  clear( matrix_(i,j) );
8089  }
8090 }
8092 //*************************************************************************************************
8093 
8094 
8095 //*************************************************************************************************
8107 template< typename MT // Type of the dense matrix
8108  , size_t... CSAs > // Compile time submatrix arguments
8110 {
8111  using blaze::clear;
8112 
8113  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
8114 
8115  const size_t jbegin( ( IsUpper_v<MT> )
8116  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
8117  ?( max( i+1UL, column() ) )
8118  :( max( i, column() ) ) )
8119  :( column() ) );
8120  const size_t jend ( ( IsLower_v<MT> )
8121  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
8122  ?( min( i, column()+columns() ) )
8123  :( min( i+1UL, column()+columns() ) ) )
8124  :( column()+columns() ) );
8125 
8126  for( size_t j=jbegin; j<jend; ++j )
8127  clear( matrix_(row()+i,j) );
8128 }
8130 //*************************************************************************************************
8131 
8132 
8133 //*************************************************************************************************
8143 template< typename MT // Type of the dense matrix
8144  , size_t... CSAs > // Compile time submatrix arguments
8145 inline bool Submatrix<MT,aligned,false,true,CSAs...>::hasOverlap() const noexcept
8146 {
8147  BLAZE_INTERNAL_ASSERT( IsSymmetric_v<MT> || IsHermitian_v<MT>, "Invalid matrix detected" );
8148 
8149  if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
8150  return false;
8151  else return true;
8152 }
8154 //*************************************************************************************************
8155 
8156 
8157 
8158 
8159 //=================================================================================================
8160 //
8161 // NUMERIC FUNCTIONS
8162 //
8163 //=================================================================================================
8164 
8165 //*************************************************************************************************
8183 template< typename MT // Type of the dense matrix
8184  , size_t... CSAs > // Compile time submatrix arguments
8185 inline Submatrix<MT,aligned,false,true,CSAs...>&
8187 {
8188  if( rows() != columns() ) {
8189  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
8190  }
8191 
8192  if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
8193  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
8194  }
8195 
8196  decltype(auto) left( derestrict( *this ) );
8197  const ResultType tmp( trans( *this ) );
8198 
8199  smpAssign( left, tmp );
8200 
8201  return *this;
8202 }
8204 //*************************************************************************************************
8205 
8206 
8207 //*************************************************************************************************
8225 template< typename MT // Type of the dense matrix
8226  , size_t... CSAs > // Compile time submatrix arguments
8227 inline Submatrix<MT,aligned,false,true,CSAs...>&
8228  Submatrix<MT,aligned,false,true,CSAs...>::ctranspose()
8229 {
8230  if( rows() != columns() ) {
8231  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
8232  }
8233 
8234  if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
8235  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
8236  }
8237 
8238  decltype(auto) left( derestrict( *this ) );
8239  const ResultType tmp( ctrans( *this ) );
8240 
8241  smpAssign( left, tmp );
8242 
8243  return *this;
8244 }
8246 //*************************************************************************************************
8247 
8248 
8249 //*************************************************************************************************
8262 template< typename MT // Type of the dense matrix
8263  , size_t... CSAs > // Compile time submatrix arguments
8264 template< typename Other > // Data type of the scalar value
8265 inline Submatrix<MT,aligned,false,true,CSAs...>&
8266  Submatrix<MT,aligned,false,true,CSAs...>::scale( const Other& scalar )
8267 {
8269 
8270  const size_t iend( row() + rows() );
8271 
8272  for( size_t i=row(); i<iend; ++i )
8273  {
8274  const size_t jbegin( ( IsUpper_v<MT> )
8275  ?( ( IsStrictlyUpper_v<MT> )
8276  ?( max( i+1UL, column() ) )
8277  :( max( i, column() ) ) )
8278  :( column() ) );
8279  const size_t jend ( ( IsLower_v<MT> )
8280  ?( ( IsStrictlyLower_v<MT> )
8281  ?( min( i, column()+columns() ) )
8282  :( min( i+1UL, column()+columns() ) ) )
8283  :( column()+columns() ) );
8284 
8285  for( size_t j=jbegin; j<jend; ++j )
8286  matrix_(i,j) *= scalar;
8287  }
8288 
8289  return *this;
8290 }
8292 //*************************************************************************************************
8293 
8294 
8295 
8296 
8297 //=================================================================================================
8298 //
8299 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
8300 //
8301 //=================================================================================================
8302 
8303 //*************************************************************************************************
8314 template< typename MT // Type of the dense matrix
8315  , size_t... CSAs > // Compile time submatrix arguments
8316 template< typename Other > // Data type of the foreign expression
8317 inline bool Submatrix<MT,aligned,false,true,CSAs...>::canAlias( const Other* alias ) const noexcept
8318 {
8319  return matrix_.isAliased( alias );
8320 }
8322 //*************************************************************************************************
8323 
8324 
8325 //*************************************************************************************************
8336 template< typename MT // Type of the dense matrix
8337  , size_t... CSAs > // Compile time submatrix arguments
8338 template< typename MT2 // Data type of the foreign dense submatrix
8339  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
8340  , bool SO2 // Storage order of the foreign dense submatrix
8341  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
8342 inline bool
8343  Submatrix<MT,aligned,false,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
8344 {
8345  return ( matrix_.isAliased( &alias->matrix_ ) &&
8346  ( row() + rows() > alias->row() ) &&
8347  ( row() < alias->row() + alias->rows() ) &&
8348  ( column() + columns() > alias->column() ) &&
8349  ( column() < alias->column() + alias->columns() ) );
8350 }
8352 //*************************************************************************************************
8353 
8354 
8355 //*************************************************************************************************
8366 template< typename MT // Type of the dense matrix
8367  , size_t... CSAs > // Compile time submatrix arguments
8368 template< typename Other > // Data type of the foreign expression
8369 inline bool Submatrix<MT,aligned,false,true,CSAs...>::isAliased( const Other* alias ) const noexcept
8370 {
8371  return matrix_.isAliased( alias );
8372 }
8374 //*************************************************************************************************
8375 
8376 
8377 //*************************************************************************************************
8388 template< typename MT // Type of the dense matrix
8389  , size_t... CSAs > // Compile time submatrix arguments
8390 template< typename MT2 // Data type of the foreign dense submatrix
8391  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
8392  , bool SO2 // Storage order of the foreign dense submatrix
8393  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
8394 inline bool
8395  Submatrix<MT,aligned,false,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
8396 {
8397  return ( matrix_.isAliased( &alias->matrix_ ) &&
8398  ( row() + rows() > alias->row() ) &&
8399  ( row() < alias->row() + alias->rows() ) &&
8400  ( column() + columns() > alias->column() ) &&
8401  ( column() < alias->column() + alias->columns() ) );
8402 }
8404 //*************************************************************************************************
8405 
8406 
8407 //*************************************************************************************************
8417 template< typename MT // Type of the dense matrix
8418  , size_t... CSAs > // Compile time submatrix arguments
8419 inline bool Submatrix<MT,aligned,false,true,CSAs...>::isAligned() const noexcept
8420 {
8421  return true;
8422 }
8424 //*************************************************************************************************
8425 
8426 
8427 //*************************************************************************************************
8438 template< typename MT // Type of the dense matrix
8439  , size_t... CSAs > // Compile time submatrix arguments
8440 inline bool Submatrix<MT,aligned,false,true,CSAs...>::canSMPAssign() const noexcept
8441 {
8442  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
8443 }
8445 //*************************************************************************************************
8446 
8447 
8448 //*************************************************************************************************
8464 template< typename MT // Type of the dense matrix
8465  , size_t... CSAs > // Compile time submatrix arguments
8466 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,false,true,CSAs...>::SIMDType
8467  Submatrix<MT,aligned,false,true,CSAs...>::load( size_t i, size_t j ) const noexcept
8468 {
8469  return loada( i, j );
8470 }
8472 //*************************************************************************************************
8473 
8474 
8475 //*************************************************************************************************
8491 template< typename MT // Type of the dense matrix
8492  , size_t... CSAs > // Compile time submatrix arguments
8493 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,false,true,CSAs...>::SIMDType
8494  Submatrix<MT,aligned,false,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
8495 {
8497 
8498  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8499  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8500  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8501  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8502 
8503  return matrix_.loada( row()+i, column()+j );
8504 }
8506 //*************************************************************************************************
8507 
8508 
8509 //*************************************************************************************************
8525 template< typename MT // Type of the dense matrix
8526  , size_t... CSAs > // Compile time submatrix arguments
8527 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,false,true,CSAs...>::SIMDType
8528  Submatrix<MT,aligned,false,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
8529 {
8531 
8532  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8533  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8534  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8535 
8536  return matrix_.loadu( row()+i, column()+j );
8537 }
8539 //*************************************************************************************************
8540 
8541 
8542 //*************************************************************************************************
8559 template< typename MT // Type of the dense matrix
8560  , size_t... CSAs > // Compile time submatrix arguments
8562  Submatrix<MT,aligned,false,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
8563 {
8564  return storea( i, j, value );
8565 }
8567 //*************************************************************************************************
8568 
8569 
8570 //*************************************************************************************************
8587 template< typename MT // Type of the dense matrix
8588  , size_t... CSAs > // Compile time submatrix arguments
8590  Submatrix<MT,aligned,false,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
8591 {
8593 
8594  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8595  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8596  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8597  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8598 
8599  return matrix_.storea( row()+i, column()+j, value );
8600 }
8602 //*************************************************************************************************
8603 
8604 
8605 //*************************************************************************************************
8622 template< typename MT // Type of the dense matrix
8623  , size_t... CSAs > // Compile time submatrix arguments
8625  Submatrix<MT,aligned,false,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
8626 {
8628 
8629  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8630  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8631  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8632 
8633  matrix_.storeu( row()+i, column()+j, value );
8634 }
8636 //*************************************************************************************************
8637 
8638 
8639 //*************************************************************************************************
8657 template< typename MT // Type of the dense matrix
8658  , size_t... CSAs > // Compile time submatrix arguments
8660  Submatrix<MT,aligned,false,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
8661 {
8663 
8664  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8665  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8666  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8667  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8668 
8669  matrix_.stream( row()+i, column()+j, value );
8670 }
8672 //*************************************************************************************************
8673 
8674 
8675 //*************************************************************************************************
8687 template< typename MT // Type of the dense matrix
8688  , size_t... CSAs > // Compile time submatrix arguments
8689 template< typename MT2 > // Type of the right-hand side dense matrix
8690 inline auto Submatrix<MT,aligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
8691  -> DisableIf_t< VectorizedAssign_v<MT2> >
8692 {
8693  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8694  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8695 
8696  const size_t jpos( columns() & size_t(-2) );
8697  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
8698 
8699  for( size_t i=0UL; i<rows(); ++i ) {
8700  for( size_t j=0UL; j<jpos; j+=2UL ) {
8701  matrix_(row()+i,column()+j ) = (~rhs)(i,j );
8702  matrix_(row()+i,column()+j+1UL) = (~rhs)(i,j+1UL);
8703  }
8704  if( jpos < columns() ) {
8705  matrix_(row()+i,column()+jpos) = (~rhs)(i,jpos);
8706  }
8707  }
8708 }
8710 //*************************************************************************************************
8711 
8712 
8713 //*************************************************************************************************
8725 template< typename MT // Type of the dense matrix
8726  , size_t... CSAs > // Compile time submatrix arguments
8727 template< typename MT2 > // Type of the right-hand side dense matrix
8728 inline auto Submatrix<MT,aligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
8729  -> EnableIf_t< VectorizedAssign_v<MT2> >
8730 {
8732 
8733  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8734  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8735 
8736  const size_t jpos( columns() & size_t(-SIMDSIZE) );
8737  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
8738 
8739  if( useStreaming &&
8740  rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
8741  !(~rhs).isAliased( &matrix_ ) )
8742  {
8743  for( size_t i=0UL; i<rows(); ++i )
8744  {
8745  size_t j( 0UL );
8746  Iterator left( begin(i) );
8747  ConstIterator_t<MT2> right( (~rhs).begin(i) );
8748 
8749  for( ; j<jpos; j+=SIMDSIZE ) {
8750  left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8751  }
8752  for( ; j<columns(); ++j ) {
8753  *left = *right; ++left; ++right;
8754  }
8755  }
8756  }
8757  else
8758  {
8759  for( size_t i=0UL; i<rows(); ++i )
8760  {
8761  size_t j( 0UL );
8762  Iterator left( begin(i) );
8763  ConstIterator_t<MT2> right( (~rhs).begin(i) );
8764 
8765  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
8766  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8767  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8768  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8769  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8770  }
8771  for( ; j<jpos; j+=SIMDSIZE ) {
8772  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8773  }
8774  for( ; j<columns(); ++j ) {
8775  *left = *right; ++left; ++right;
8776  }
8777  }
8778  }
8779 }
8781 //*************************************************************************************************
8782 
8783 
8784 //*************************************************************************************************
8796 template< typename MT // Type of the dense matrix
8797  , size_t... CSAs > // Compile time submatrix arguments
8798 template< typename MT2 > // Type of the right-hand side dense matrix
8799 inline void Submatrix<MT,aligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
8800 {
8802 
8803  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8804  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8805 
8806  constexpr size_t block( BLOCK_SIZE );
8807 
8808  for( size_t ii=0UL; ii<rows(); ii+=block ) {
8809  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
8810  for( size_t jj=0UL; jj<columns(); jj+=block ) {
8811  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
8812  for( size_t i=ii; i<iend; ++i ) {
8813  for( size_t j=jj; j<jend; ++j ) {
8814  matrix_(row()+i,column()+j) = (~rhs)(i,j);
8815  }
8816  }
8817  }
8818  }
8819 }
8821 //*************************************************************************************************
8822 
8823 
8824 //*************************************************************************************************
8836 template< typename MT // Type of the dense matrix
8837  , size_t... CSAs > // Compile time submatrix arguments
8838 template< typename MT2 > // Type of the right-hand side sparse matrix
8839 inline void Submatrix<MT,aligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
8840 {
8841  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8842  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8843 
8844  for( size_t i=0UL; i<rows(); ++i )
8845  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
8846  matrix_(row()+i,column()+element->index()) = element->value();
8847 }
8849 //*************************************************************************************************
8850 
8851 
8852 //*************************************************************************************************
8864 template< typename MT // Type of the dense matrix
8865  , size_t... CSAs > // Compile time submatrix arguments
8866 template< typename MT2 > // Type of the right-hand side sparse matrix
8867 inline void Submatrix<MT,aligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
8868 {
8870 
8871  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8872  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8873 
8874  for( size_t j=0UL; j<columns(); ++j )
8875  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
8876  matrix_(row()+element->index(),column()+j) = element->value();
8877 }
8879 //*************************************************************************************************
8880 
8881 
8882 //*************************************************************************************************
8894 template< typename MT // Type of the dense matrix
8895  , size_t... CSAs > // Compile time submatrix arguments
8896 template< typename MT2 > // Type of the right-hand side dense matrix
8897 inline auto Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
8898  -> DisableIf_t< VectorizedAddAssign_v<MT2> >
8899 {
8900  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8901  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8902 
8903  const size_t jpos( columns() & size_t(-2) );
8904  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
8905 
8906  for( size_t i=0UL; i<rows(); ++i )
8907  {
8908  if( IsDiagonal_v<MT2> ) {
8909  matrix_(row()+i,column()+i) += (~rhs)(i,i);
8910  }
8911  else {
8912  for( size_t j=0UL; j<jpos; j+=2UL ) {
8913  matrix_(row()+i,column()+j ) += (~rhs)(i,j );
8914  matrix_(row()+i,column()+j+1UL) += (~rhs)(i,j+1UL);
8915  }
8916  if( jpos < columns() ) {
8917  matrix_(row()+i,column()+jpos) += (~rhs)(i,jpos);
8918  }
8919  }
8920  }
8921 }
8923 //*************************************************************************************************
8924 
8925 
8926 //*************************************************************************************************
8938 template< typename MT // Type of the dense matrix
8939  , size_t... CSAs > // Compile time submatrix arguments
8940 template< typename MT2 > // Type of the right-hand side dense matrix
8941 inline auto Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
8942  -> EnableIf_t< VectorizedAddAssign_v<MT2> >
8943 {
8945 
8946  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8947  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8948 
8949  for( size_t i=0UL; i<rows(); ++i )
8950  {
8951  const size_t jbegin( ( IsUpper_v<MT2> )
8952  ?( ( IsStrictlyUpper_v<MT2> ? i+1UL : i ) & size_t(-SIMDSIZE) )
8953  :( 0UL ) );
8954  const size_t jend ( ( IsLower_v<MT2> )
8955  ?( IsStrictlyLower_v<MT2> ? i : i+1UL )
8956  :( columns() ) );
8957  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
8958 
8959  const size_t jpos( jend & size_t(-SIMDSIZE) );
8960  BLAZE_INTERNAL_ASSERT( ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
8961 
8962  size_t j( jbegin );
8963  Iterator left( begin(i) + jbegin );
8964  ConstIterator_t<MT2> right( (~rhs).begin(i) + jbegin );
8965 
8966  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
8967  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8968  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8969  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8970  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8971  }
8972  for( ; j<jpos; j+=SIMDSIZE ) {
8973  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8974  }
8975  for( ; j<jend; ++j ) {
8976  *left += *right; ++left; ++right;
8977  }
8978  }
8979 }
8981 //*************************************************************************************************
8982 
8983 
8984 //*************************************************************************************************
8996 template< typename MT // Type of the dense matrix
8997  , size_t... CSAs > // Compile time submatrix arguments
8998 template< typename MT2 > // Type of the right-hand side dense matrix
8999 inline void Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
9000 {
9002 
9003  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9004  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9005 
9006  constexpr size_t block( BLOCK_SIZE );
9007 
9008  for( size_t ii=0UL; ii<rows(); ii+=block ) {
9009  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
9010  for( size_t jj=0UL; jj<columns(); jj+=block ) {
9011  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
9012  for( size_t i=ii; i<iend; ++i ) {
9013  for( size_t j=jj; j<jend; ++j ) {
9014  matrix_(row()+i,column()+j) += (~rhs)(i,j);
9015  }
9016  }
9017  }
9018  }
9019 }
9021 //*************************************************************************************************
9022 
9023 
9024 //*************************************************************************************************
9036 template< typename MT // Type of the dense matrix
9037  , size_t... CSAs > // Compile time submatrix arguments
9038 template< typename MT2 > // Type of the right-hand side sparse matrix
9039 inline void Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
9040 {
9041  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9042  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9043 
9044  for( size_t i=0UL; i<rows(); ++i )
9045  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
9046  matrix_(row()+i,column()+element->index()) += element->value();
9047 }
9049 //*************************************************************************************************
9050 
9051 
9052 //*************************************************************************************************
9064 template< typename MT // Type of the dense matrix
9065  , size_t... CSAs > // Compile time submatrix arguments
9066 template< typename MT2 > // Type of the right-hand side sparse matrix
9067 inline void Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
9068 {
9070 
9071  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9072  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9073 
9074  for( size_t j=0UL; j<columns(); ++j )
9075  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
9076  matrix_(row()+element->index(),column()+j) += element->value();
9077 }
9079 //*************************************************************************************************
9080 
9081 
9082 //*************************************************************************************************
9094 template< typename MT // Type of the dense matrix
9095  , size_t... CSAs > // Compile time submatrix arguments
9096 template< typename MT2 > // Type of the right-hand side dense matrix
9097 inline auto Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
9098  -> DisableIf_t< VectorizedSubAssign_v<MT2> >
9099 {
9100  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9101  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9102 
9103  const size_t jpos( columns() & size_t(-2) );
9104  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
9105 
9106  for( size_t i=0UL; i<rows(); ++i )
9107  {
9108  if( IsDiagonal_v<MT2> ) {
9109  matrix_(row()+i,column()+i) -= (~rhs)(i,i);
9110  }
9111  else {
9112  for( size_t j=0UL; j<jpos; j+=2UL ) {
9113  matrix_(row()+i,column()+j ) -= (~rhs)(i,j );
9114  matrix_(row()+i,column()+j+1UL) -= (~rhs)(i,j+1UL);
9115  }
9116  if( jpos < columns() ) {
9117  matrix_(row()+i,column()+jpos) -= (~rhs)(i,jpos);
9118  }
9119  }
9120  }
9121 }
9123 //*************************************************************************************************
9124 
9125 
9126 //*************************************************************************************************
9138 template< typename MT // Type of the dense matrix
9139  , size_t... CSAs > // Compile time submatrix arguments
9140 template< typename MT2 > // Type of the right-hand side dense matrix
9141 inline auto Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
9142  -> EnableIf_t< VectorizedSubAssign_v<MT2> >
9143 {
9145 
9146  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9147  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9148 
9149  for( size_t i=0UL; i<rows(); ++i )
9150  {
9151  const size_t jbegin( ( IsUpper_v<MT2> )
9152  ?( ( IsStrictlyUpper_v<MT2> ? i+1UL : i ) & size_t(-SIMDSIZE) )
9153  :( 0UL ) );
9154  const size_t jend ( ( IsLower_v<MT2> )
9155  ?( IsStrictlyLower_v<MT2> ? i : i+1UL )
9156  :( columns() ) );
9157  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
9158 
9159  const size_t jpos( jend & size_t(-SIMDSIZE) );
9160  BLAZE_INTERNAL_ASSERT( ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
9161 
9162  size_t j( jbegin );
9163  Iterator left( begin(i) + jbegin );
9164  ConstIterator_t<MT2> right( (~rhs).begin(i) + jbegin );
9165 
9166  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
9167  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9168  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9169  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9170  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9171  }
9172  for( ; j<jpos; j+=SIMDSIZE ) {
9173  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9174  }
9175  for( ; j<jend; ++j ) {
9176  *left -= *right; ++left; ++right;
9177  }
9178  }
9179 }
9181 //*************************************************************************************************
9182 
9183 
9184 //*************************************************************************************************
9196 template< typename MT // Type of the dense matrix
9197  , size_t... CSAs > // Compile time submatrix arguments
9198 template< typename MT2 > // Type of the right-hand side dense matrix
9199 inline void Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
9200 {
9202 
9203  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9204  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9205 
9206  constexpr size_t block( BLOCK_SIZE );
9207 
9208  for( size_t ii=0UL; ii<rows(); ii+=block ) {
9209  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
9210  for( size_t jj=0UL; jj<columns(); jj+=block ) {
9211  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
9212  for( size_t i=ii; i<iend; ++i ) {
9213  for( size_t j=jj; j<jend; ++j ) {
9214  matrix_(row()+i,column()+j) -= (~rhs)(i,j);
9215  }
9216  }
9217  }
9218  }
9219 }
9221 //*************************************************************************************************
9222 
9223 
9224 //*************************************************************************************************
9236 template< typename MT // Type of the dense matrix
9237  , size_t... CSAs > // Compile time submatrix arguments
9238 template< typename MT2 > // Type of the right-hand side sparse matrix
9239 inline void Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
9240 {
9241  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9242  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9243 
9244  for( size_t i=0UL; i<rows(); ++i )
9245  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
9246  matrix_(row()+i,column()+element->index()) -= element->value();
9247 }
9249 //*************************************************************************************************
9250 
9251 
9252 //*************************************************************************************************
9264 template< typename MT // Type of the dense matrix
9265  , size_t... CSAs > // Compile time submatrix arguments
9266 template< typename MT2 > // Type of the right-hand side sparse matrix
9267 inline void Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
9268 {
9270 
9271  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9272  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9273 
9274  for( size_t j=0UL; j<columns(); ++j )
9275  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
9276  matrix_(row()+element->index(),column()+j) -= element->value();
9277 }
9279 //*************************************************************************************************
9280 
9281 
9282 //*************************************************************************************************
9294 template< typename MT // Type of the dense matrix
9295  , size_t... CSAs > // Compile time submatrix arguments
9296 template< typename MT2 > // Type of the right-hand side dense matrix
9297 inline auto Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
9298  -> DisableIf_t< VectorizedSchurAssign_v<MT2> >
9299 {
9300  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9301  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9302 
9303  const size_t jpos( columns() & size_t(-2) );
9304  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
9305 
9306  for( size_t i=0UL; i<rows(); ++i ) {
9307  for( size_t j=0UL; j<jpos; j+=2UL ) {
9308  matrix_(row()+i,column()+j ) *= (~rhs)(i,j );
9309  matrix_(row()+i,column()+j+1UL) *= (~rhs)(i,j+1UL);
9310  }
9311  if( jpos < columns() ) {
9312  matrix_(row()+i,column()+jpos) *= (~rhs)(i,jpos);
9313  }
9314  }
9315 }
9317 //*************************************************************************************************
9318 
9319 
9320 //*************************************************************************************************
9332 template< typename MT // Type of the dense matrix
9333  , size_t... CSAs > // Compile time submatrix arguments
9334 template< typename MT2 > // Type of the right-hand side dense matrix
9335 inline auto Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
9336  -> EnableIf_t< VectorizedSchurAssign_v<MT2> >
9337 {
9339 
9340  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9341  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9342 
9343  for( size_t i=0UL; i<rows(); ++i )
9344  {
9345  const size_t jpos( columns() & size_t(-SIMDSIZE) );
9346  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
9347 
9348  size_t j( 0UL );
9349  Iterator left( begin(i) );
9350  ConstIterator_t<MT2> right( (~rhs).begin(i) );
9351 
9352  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
9353  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9354  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9355  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9356  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9357  }
9358  for( ; j<jpos; j+=SIMDSIZE ) {
9359  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9360  }
9361  for( ; j<columns(); ++j ) {
9362  *left *= *right; ++left; ++right;
9363  }
9364  }
9365 }
9367 //*************************************************************************************************
9368 
9369 
9370 //*************************************************************************************************
9382 template< typename MT // Type of the dense matrix
9383  , size_t... CSAs > // Compile time submatrix arguments
9384 template< typename MT2 > // Type of the right-hand side dense matrix
9385 inline void Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
9386 {
9388 
9389  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9390  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9391 
9392  constexpr size_t block( BLOCK_SIZE );
9393 
9394  for( size_t ii=0UL; ii<rows(); ii+=block ) {
9395  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
9396  for( size_t jj=0UL; jj<columns(); jj+=block ) {
9397  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
9398  for( size_t i=ii; i<iend; ++i ) {
9399  for( size_t j=jj; j<jend; ++j ) {
9400  matrix_(row()+i,column()+j) *= (~rhs)(i,j);
9401  }
9402  }
9403  }
9404  }
9405 }
9407 //*************************************************************************************************
9408 
9409 
9410 //*************************************************************************************************
9422 template< typename MT // Type of the dense matrix
9423  , size_t... CSAs > // Compile time submatrix arguments
9424 template< typename MT2 > // Type of the right-hand side sparse matrix
9425 inline void Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
9426 {
9427  using blaze::reset;
9428 
9429  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9430  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9431 
9432  for( size_t i=0UL; i<rows(); ++i )
9433  {
9434  size_t j( 0UL );
9435 
9436  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element ) {
9437  for( ; j<element->index(); ++j )
9438  reset( matrix_(row()+i,column()+j) );
9439  matrix_(row()+i,column()+j) *= element->value();
9440  ++j;
9441  }
9442 
9443  for( ; j<columns(); ++j ) {
9444  reset( matrix_(row()+i,column()+j) );
9445  }
9446  }
9447 }
9449 //*************************************************************************************************
9450 
9451 
9452 //*************************************************************************************************
9464 template< typename MT // Type of the dense matrix
9465  , size_t... CSAs > // Compile time submatrix arguments
9466 template< typename MT2 > // Type of the right-hand side sparse matrix
9467 inline void Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
9468 {
9469  using blaze::reset;
9470 
9472 
9473  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9474  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9475 
9476  for( size_t j=0UL; j<columns(); ++j )
9477  {
9478  size_t i( 0UL );
9479 
9480  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element ) {
9481  for( ; i<element->index(); ++i )
9482  reset( matrix_(row()+i,column()+j) );
9483  matrix_(row()+element->index(),column()+j) *= element->value();
9484  ++i;
9485  }
9486 
9487  for( ; i<rows(); ++i ) {
9488  reset( matrix_(row()+i,column()+j) );
9489  }
9490  }
9491 }
9493 //*************************************************************************************************
9494 
9495 
9496 
9497 
9498 
9499 
9500 
9501 
9502 //=================================================================================================
9503 //
9504 // CLASS TEMPLATE SPECIALIZATION FOR ALIGNED COLUMN-MAJOR DENSE SUBMATRICES
9505 //
9506 //=================================================================================================
9507 
9508 //*************************************************************************************************
9516 template< typename MT // Type of the dense matrix
9517  , size_t... CSAs > // Compile time submatrix arguments
9518 class Submatrix<MT,aligned,true,true,CSAs...>
9519  : public View< DenseMatrix< Submatrix<MT,aligned,true,true,CSAs...>, true > >
9520  , private SubmatrixData<CSAs...>
9521 {
9522  private:
9523  //**Type definitions****************************************************************************
9524  using DataType = SubmatrixData<CSAs...>;
9525  using Operand = If_t< IsExpression_v<MT>, MT, MT& >;
9526  //**********************************************************************************************
9527 
9528  //**********************************************************************************************
9530  template< typename MT1, typename MT2 >
9531  static constexpr bool EnforceEvaluation_v =
9532  ( IsRestricted_v<MT1> && RequiresEvaluation_v<MT2> );
9533  //**********************************************************************************************
9534 
9535  public:
9536  //**Type definitions****************************************************************************
9538  using This = Submatrix<MT,aligned,true,true,CSAs...>;
9539 
9540  using BaseType = DenseMatrix<This,true>;
9541  using ViewedType = MT;
9542  using ResultType = SubmatrixTrait_t<MT,CSAs...>;
9543  using OppositeType = OppositeType_t<ResultType>;
9544  using TransposeType = TransposeType_t<ResultType>;
9545  using ElementType = ElementType_t<MT>;
9546  using SIMDType = SIMDTrait_t<ElementType>;
9547  using ReturnType = ReturnType_t<MT>;
9548  using CompositeType = const Submatrix&;
9549 
9551  using ConstReference = ConstReference_t<MT>;
9552 
9554  using Reference = If_t< IsConst_v<MT>, ConstReference, Reference_t<MT> >;
9555 
9557  using ConstPointer = ConstPointer_t<MT>;
9558 
9560  using Pointer = If_t< IsConst_v<MT> || !HasMutableDataAccess_v<MT>, ConstPointer, Pointer_t<MT> >;
9561 
9563  using ConstIterator = ConstIterator_t<MT>;
9564 
9566  using Iterator = If_t< IsConst_v<MT>, ConstIterator, Iterator_t<MT> >;
9567  //**********************************************************************************************
9568 
9569  //**Compilation flags***************************************************************************
9571  static constexpr bool simdEnabled = MT::simdEnabled;
9572 
9574  static constexpr bool smpAssignable = MT::smpAssignable;
9575 
9577  static constexpr bool compileTimeArgs = DataType::compileTimeArgs;
9578  //**********************************************************************************************
9579 
9580  //**Constructors********************************************************************************
9583  template< typename... RSAs >
9584  explicit inline Submatrix( MT& matrix, RSAs... args );
9585 
9586  Submatrix( const Submatrix& ) = default;
9588  //**********************************************************************************************
9589 
9590  //**Destructor**********************************************************************************
9593  ~Submatrix() = default;
9595  //**********************************************************************************************
9596 
9597  //**Data access functions***********************************************************************
9600  inline Reference operator()( size_t i, size_t j );
9601  inline ConstReference operator()( size_t i, size_t j ) const;
9602  inline Reference at( size_t i, size_t j );
9603  inline ConstReference at( size_t i, size_t j ) const;
9604  inline Pointer data () noexcept;
9605  inline ConstPointer data () const noexcept;
9606  inline Pointer data ( size_t j ) noexcept;
9607  inline ConstPointer data ( size_t j ) const noexcept;
9608  inline Iterator begin ( size_t j );
9609  inline ConstIterator begin ( size_t j ) const;
9610  inline ConstIterator cbegin( size_t j ) const;
9611  inline Iterator end ( size_t j );
9612  inline ConstIterator end ( size_t j ) const;
9613  inline ConstIterator cend ( size_t j ) const;
9615  //**********************************************************************************************
9616 
9617  //**Assignment operators************************************************************************
9620  inline Submatrix& operator=( const ElementType& rhs );
9621  inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
9622  inline Submatrix& operator=( const Submatrix& rhs );
9623 
9624  template< typename MT2, bool SO >
9625  inline Submatrix& operator=( const Matrix<MT2,SO>& rhs );
9626 
9627  template< typename MT2, bool SO >
9628  inline auto operator+=( const Matrix<MT2,SO>& rhs )
9629  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9630 
9631  template< typename MT2, bool SO >
9632  inline auto operator+=( const Matrix<MT2,SO>& rhs )
9633  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9634 
9635  template< typename MT2, bool SO >
9636  inline auto operator-=( const Matrix<MT2,SO>& rhs )
9637  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9638 
9639  template< typename MT2, bool SO >
9640  inline auto operator-=( const Matrix<MT2,SO>& rhs )
9641  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9642 
9643  template< typename MT2, bool SO >
9644  inline auto operator%=( const Matrix<MT2,SO>& rhs )
9645  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9646 
9647  template< typename MT2, bool SO >
9648  inline auto operator%=( const Matrix<MT2,SO>& rhs )
9649  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9651  //**********************************************************************************************
9652 
9653  //**Utility functions***************************************************************************
9656  using DataType::row;
9657  using DataType::column;
9658  using DataType::rows;
9659  using DataType::columns;
9660 
9661  inline MT& operand() noexcept;
9662  inline const MT& operand() const noexcept;
9663 
9664  inline size_t spacing() const noexcept;
9665  inline size_t capacity() const noexcept;
9666  inline size_t capacity( size_t i ) const noexcept;
9667  inline size_t nonZeros() const;
9668  inline size_t nonZeros( size_t i ) const;
9669  inline void reset();
9670  inline void reset( size_t i );
9672  //**********************************************************************************************
9673 
9674  //**Numeric functions***************************************************************************
9677  inline Submatrix& transpose();
9678  inline Submatrix& ctranspose();
9679 
9680  template< typename Other > inline Submatrix& scale( const Other& scalar );
9682  //**********************************************************************************************
9683 
9684  private:
9685  //**********************************************************************************************
9687  template< typename MT2 >
9688  static constexpr bool VectorizedAssign_v =
9689  ( useOptimizedKernels &&
9690  simdEnabled && MT2::simdEnabled &&
9691  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > );
9692  //**********************************************************************************************
9693 
9694  //**********************************************************************************************
9696  template< typename MT2 >
9697  static constexpr bool VectorizedAddAssign_v =
9698  ( useOptimizedKernels &&
9699  simdEnabled && MT2::simdEnabled &&
9700  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > &&
9701  HasSIMDAdd_v< ElementType, ElementType_t<MT2> > &&
9702  !IsDiagonal_v<MT2> );
9703  //**********************************************************************************************
9704 
9705  //**********************************************************************************************
9707  template< typename MT2 >
9708  static constexpr bool VectorizedSubAssign_v =
9709  ( useOptimizedKernels &&
9710  simdEnabled && MT2::simdEnabled &&
9711  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > &&
9712  HasSIMDSub_v< ElementType, ElementType_t<MT2> > &&
9713  !IsDiagonal_v<MT2> );
9714  //**********************************************************************************************
9715 
9716  //**********************************************************************************************
9718  template< typename MT2 >
9719  static constexpr bool VectorizedSchurAssign_v =
9720  ( useOptimizedKernels &&
9721  simdEnabled && MT2::simdEnabled &&
9722  IsSIMDCombinable_v< ElementType, ElementType_t<MT2> > &&
9723  HasSIMDMult_v< ElementType, ElementType_t<MT2> > );
9724  //**********************************************************************************************
9725 
9726  //**SIMD properties*****************************************************************************
9728  static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
9729  //**********************************************************************************************
9730 
9731  public:
9732  //**Expression template evaluation functions****************************************************
9735  template< typename Other >
9736  inline bool canAlias( const Other* alias ) const noexcept;
9737 
9738  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
9739  inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
9740 
9741  template< typename Other >
9742  inline bool isAliased( const Other* alias ) const noexcept;
9743 
9744  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
9745  inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
9746 
9747  inline bool isAligned () const noexcept;
9748  inline bool canSMPAssign() const noexcept;
9749 
9750  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
9751  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
9752  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
9753 
9754  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
9755  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
9756  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
9757  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
9758 
9759  template< typename MT2 >
9760  inline auto assign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT2> >;
9761 
9762  template< typename MT2 >
9763  inline auto assign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT2> >;
9764 
9765  template< typename MT2 > inline void assign( const DenseMatrix<MT2,false>& rhs );
9766  template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
9767  template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
9768 
9769  template< typename MT2 >
9770  inline auto addAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT2> >;
9771 
9772  template< typename MT2 >
9773  inline auto addAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT2> >;
9774 
9775  template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,false>& rhs );
9776  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
9777  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
9778 
9779  template< typename MT2 >
9780  inline auto subAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT2> >;
9781 
9782  template< typename MT2 >
9783  inline auto subAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT2> >;
9784 
9785  template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,false>& rhs );
9786  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
9787  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
9788 
9789  template< typename MT2 >
9790  inline auto schurAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT2> >;
9791 
9792  template< typename MT2 >
9793  inline auto schurAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT2> >;
9794 
9795  template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,false>& rhs );
9796  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
9797  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
9799  //**********************************************************************************************
9800 
9801  private:
9802  //**Utility functions***************************************************************************
9805  inline bool hasOverlap() const noexcept;
9807  //**********************************************************************************************
9808 
9809  //**Member variables****************************************************************************
9812  Operand matrix_;
9813 
9814  //**********************************************************************************************
9815 
9816  //**Friend declarations*************************************************************************
9817  template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
9818  //**********************************************************************************************
9819 
9820  //**Compile time checks*************************************************************************
9828  //**********************************************************************************************
9829 };
9831 //*************************************************************************************************
9832 
9833 
9834 
9835 
9836 //=================================================================================================
9837 //
9838 // CONSTRUCTORS
9839 //
9840 //=================================================================================================
9841 
9842 //*************************************************************************************************
9855 template< typename MT // Type of the dense matrix
9856  , size_t... CSAs > // Compile time submatrix arguments
9857 template< typename... RSAs > // Runtime submatrix arguments
9858 inline Submatrix<MT,aligned,true,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
9859  : DataType( args... ) // Base class initialization
9860  , matrix_ ( matrix ) // The matrix containing the submatrix
9861 {
9862  if( !Contains_v< TypeList<RSAs...>, Unchecked > )
9863  {
9864  if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
9865  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
9866  }
9867 
9868  if( simdEnabled && IsContiguous_v<MT> &&
9869  ( !checkAlignment( data() ) ||
9870  ( columns() > 1UL && matrix_.spacing() % SIMDSIZE != 0UL ) ) ) {
9871  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix alignment" );
9872  }
9873  }
9874  else
9875  {
9876  BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
9877  BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
9878 
9879  BLAZE_USER_ASSERT( !simdEnabled || !IsContiguous_v<MT> || checkAlignment( data() ), "Invalid submatrix alignment" );
9880  BLAZE_USER_ASSERT( !simdEnabled || !IsContiguous_v<MT> || columns() <= 1UL || matrix_.spacing() % SIMDSIZE == 0UL, "Invalid submatrix alignment" );
9881  }
9882 }
9884 //*************************************************************************************************
9885 
9886 
9887 
9888 
9889 //=================================================================================================
9890 //
9891 // DATA ACCESS FUNCTIONS
9892 //
9893 //=================================================================================================
9894 
9895 //*************************************************************************************************
9906 template< typename MT // Type of the dense matrix
9907  , size_t... CSAs > // Compile time submatrix arguments
9908 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Reference
9909  Submatrix<MT,aligned,true,true,CSAs...>::operator()( size_t i, size_t j )
9910 {
9911  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
9912  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
9913 
9914  return matrix_(row()+i,column()+j);
9915 }
9917 //*************************************************************************************************
9918 
9919 
9920 //*************************************************************************************************
9931 template< typename MT // Type of the dense matrix
9932  , size_t... CSAs > // Compile time submatrix arguments
9933 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstReference
9934  Submatrix<MT,aligned,true,true,CSAs...>::operator()( size_t i, size_t j ) const
9935 {
9936  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
9937  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
9938 
9939  return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
9940 }
9942 //*************************************************************************************************
9943 
9944 
9945 //*************************************************************************************************
9957 template< typename MT // Type of the dense matrix
9958  , size_t... CSAs > // Compile time submatrix arguments
9959 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Reference
9960  Submatrix<MT,aligned,true,true,CSAs...>::at( size_t i, size_t j )
9961 {
9962  if( i >= rows() ) {
9963  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
9964  }
9965  if( j >= columns() ) {
9966  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
9967  }
9968  return (*this)(i,j);
9969 }
9971 //*************************************************************************************************
9972 
9973 
9974 //*************************************************************************************************
9986 template< typename MT // Type of the dense matrix
9987  , size_t... CSAs > // Compile time submatrix arguments
9988 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstReference
9989  Submatrix<MT,aligned,true,true,CSAs...>::at( size_t i, size_t j ) const
9990 {
9991  if( i >= rows() ) {
9992  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
9993  }
9994  if( j >= columns() ) {
9995  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
9996  }
9997  return (*this)(i,j);
9998 }
10000 //*************************************************************************************************
10001 
10002 
10003 //*************************************************************************************************
10013 template< typename MT // Type of the dense matrix
10014  , size_t... CSAs > // Compile time submatrix arguments
10015 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Pointer
10017 {
10018  return matrix_.data() + row() + column()*spacing();
10019 }
10021 //*************************************************************************************************
10022 
10023 
10024 //*************************************************************************************************
10034 template< typename MT // Type of the dense matrix
10035  , size_t... CSAs > // Compile time submatrix arguments
10036 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstPointer
10038 {
10039  return matrix_.data() + row() + column()*spacing();
10040 }
10042 //*************************************************************************************************
10043 
10044 
10045 //*************************************************************************************************
10054 template< typename MT // Type of the dense matrix
10055  , size_t... CSAs > // Compile time submatrix arguments
10056 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Pointer
10058 {
10059  return matrix_.data() + row() + (column()+j)*spacing();
10060 }
10062 //*************************************************************************************************
10063 
10064 
10065 //*************************************************************************************************
10074 template< typename MT // Type of the dense matrix
10075  , size_t... CSAs > // Compile time submatrix arguments
10076 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstPointer
10077  Submatrix<MT,aligned,true,true,CSAs...>::data( size_t j ) const noexcept
10078 {
10079  return matrix_.data() + row() + (column()+j)*spacing();
10080 }
10082 //*************************************************************************************************
10083 
10084 
10085 //*************************************************************************************************
10092 template< typename MT // Type of the dense matrix
10093  , size_t... CSAs > // Compile time submatrix arguments
10094 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Iterator
10096 {
10097  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10098  return ( matrix_.begin( column() + j ) + row() );
10099 }
10101 //*************************************************************************************************
10102 
10103 
10104 //*************************************************************************************************
10111 template< typename MT // Type of the dense matrix
10112  , size_t... CSAs > // Compile time submatrix arguments
10113 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10115 {
10116  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10117  return ( matrix_.cbegin( column() + j ) + row() );
10118 }
10120 //*************************************************************************************************
10121 
10122 
10123 //*************************************************************************************************
10130 template< typename MT // Type of the dense matrix
10131  , size_t... CSAs > // Compile time submatrix arguments
10132 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10134 {
10135  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10136  return ( matrix_.cbegin( column() + j ) + row() );
10137 }
10139 //*************************************************************************************************
10140 
10141 
10142 //*************************************************************************************************
10149 template< typename MT // Type of the dense matrix
10150  , size_t... CSAs > // Compile time submatrix arguments
10151 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Iterator
10153 {
10154  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10155  return ( matrix_.begin( column() + j ) + row() + rows() );
10156 }
10158 //*************************************************************************************************
10159 
10160 
10161 //*************************************************************************************************
10168 template< typename MT // Type of the dense matrix
10169  , size_t... CSAs > // Compile time submatrix arguments
10170 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10172 {
10173  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10174  return ( matrix_.cbegin( column() + j ) + row() + rows() );
10175 }
10177 //*************************************************************************************************
10178 
10179 
10180 //*************************************************************************************************
10187 template< typename MT // Type of the dense matrix
10188  , size_t... CSAs > // Compile time submatrix arguments
10189 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10191 {
10192  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10193  return ( matrix_.cbegin( column() + j ) + row() + rows() );
10194 }
10196 //*************************************************************************************************
10197 
10198 
10199 
10200 
10201 //=================================================================================================
10202 //
10203 // ASSIGNMENT OPERATORS
10204 //
10205 //=================================================================================================
10206 
10207 //*************************************************************************************************
10218 template< typename MT // Type of the dense matrix
10219  , size_t... CSAs > // Compile time submatrix arguments
10220 inline Submatrix<MT,aligned,true,true,CSAs...>&
10221  Submatrix<MT,aligned,true,true,CSAs...>::operator=( const ElementType& rhs )
10222 {
10223  const size_t jend( column() + columns() );
10224  decltype(auto) left( derestrict( matrix_ ) );
10225 
10226  for( size_t j=column(); j<jend; ++j )
10227  {
10228  const size_t ibegin( ( IsLower_v<MT> )
10229  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
10230  ?( max( j+1UL, row() ) )
10231  :( max( j, row() ) ) )
10232  :( row() ) );
10233  const size_t iend ( ( IsUpper_v<MT> )
10234  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
10235  ?( min( j, row()+rows() ) )
10236  :( min( j+1UL, row()+rows() ) ) )
10237  :( row()+rows() ) );
10238 
10239  for( size_t i=ibegin; i<iend; ++i ) {
10240  if( !IsRestricted_v<MT> || IsTriangular_v<MT> || trySet( matrix_, i, j, rhs ) )
10241  left(i,j) = rhs;
10242  }
10243  }
10244 
10245  return *this;
10246 }
10248 //*************************************************************************************************
10249 
10250 
10251 //*************************************************************************************************
10267 template< typename MT // Type of the dense matrix
10268  , size_t... CSAs > // Compile time submatrix arguments
10269 inline Submatrix<MT,aligned,true,true,CSAs...>&
10270  Submatrix<MT,aligned,true,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
10271 {
10272  using blaze::reset;
10273 
10274  if( list.size() != rows() ) {
10275  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
10276  }
10277 
10278  if( IsRestricted_v<MT> ) {
10279  const InitializerMatrix<ElementType> tmp( list, columns() );
10280  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10281  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10282  }
10283  }
10284 
10285  decltype(auto) left( derestrict( *this ) );
10286  size_t i( 0UL );
10287 
10288  for( const auto& rowList : list ) {
10289  size_t j( 0UL );
10290  for( const auto& element : rowList ) {
10291  left(i,j) = element;
10292  ++j;
10293  }
10294  for( ; j<columns(); ++j ) {
10295  reset( left(i,j) );
10296  }
10297  ++i;
10298  }
10299 
10300  return *this;
10301 }
10303 //*************************************************************************************************
10304 
10305 
10306 //*************************************************************************************************
10321 template< typename MT // Type of the dense matrix
10322  , size_t... CSAs > // Compile time submatrix arguments
10323 inline Submatrix<MT,aligned,true,true,CSAs...>&
10324  Submatrix<MT,aligned,true,true,CSAs...>::operator=( const Submatrix& rhs )
10325 {
10328 
10329  if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
10330  return *this;
10331 
10332  if( rows() != rhs.rows() || columns() != rhs.columns() ) {
10333  BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
10334  }
10335 
10336  if( !tryAssign( matrix_, rhs, row(), column() ) ) {
10337  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10338  }
10339 
10340  decltype(auto) left( derestrict( *this ) );
10341 
10342  if( rhs.canAlias( &matrix_ ) ) {
10343  const ResultType tmp( rhs );
10344  smpAssign( left, tmp );
10345  }
10346  else {
10347  smpAssign( left, rhs );
10348  }
10349 
10350  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10351 
10352  return *this;
10353 }
10355 //*************************************************************************************************
10356 
10357 
10358 //*************************************************************************************************
10372 template< typename MT // Type of the dense matrix
10373  , size_t... CSAs > // Compile time submatrix arguments
10374 template< typename MT2 // Type of the right-hand side matrix
10375  , bool SO > // Storage order of the right-hand side matrix
10376 inline Submatrix<MT,aligned,true,true,CSAs...>&
10377  Submatrix<MT,aligned,true,true,CSAs...>::operator=( const Matrix<MT2,SO>& rhs )
10378 {
10379  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10380 
10381  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10382  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10383  }
10384 
10385  using Right = If_t< IsRestricted_v<MT>, CompositeType_t<MT2>, const MT2& >;
10386  Right right( ~rhs );
10387 
10388  if( !tryAssign( matrix_, right, row(), column() ) ) {
10389  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10390  }
10391 
10392  decltype(auto) left( derestrict( *this ) );
10393 
10394  if( IsReference_v<Right> && right.canAlias( &matrix_ ) ) {
10395  const ResultType_t<MT2> tmp( right );
10396  if( IsSparseMatrix_v<MT2> )
10397  reset();
10398  smpAssign( left, tmp );
10399  }
10400  else {
10401  if( IsSparseMatrix_v<MT2> )
10402  reset();
10403  smpAssign( left, right );
10404  }
10405 
10406  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10407 
10408  return *this;
10409 }
10411 //*************************************************************************************************
10412 
10413 
10414 //*************************************************************************************************
10428 template< typename MT // Type of the dense matrix
10429  , size_t... CSAs > // Compile time submatrix arguments
10430 template< typename MT2 // Type of the right-hand side matrix
10431  , bool SO > // Storage order of the right-hand side matrix
10432 inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
10433  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10434 {
10437  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10438 
10439  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
10440 
10443 
10444  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10445  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10446  }
10447 
10448  if( !tryAddAssign( matrix_, ~rhs, row(), column() ) ) {
10449  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10450  }
10451 
10452  decltype(auto) left( derestrict( *this ) );
10453 
10454  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
10455  (~rhs).canAlias( &matrix_ ) ) {
10456  const AddType tmp( *this + (~rhs) );
10457  smpAssign( left, tmp );
10458  }
10459  else {
10460  smpAddAssign( left, ~rhs );
10461  }
10462 
10463  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10464 
10465  return *this;
10466 }
10468 //*************************************************************************************************
10469 
10470 
10471 //*************************************************************************************************
10485 template< typename MT // Type of the dense matrix
10486  , size_t... CSAs > // Compile time submatrix arguments
10487 template< typename MT2 // Type of the right-hand side matrix
10488  , bool SO > // Storage order of the right-hand side matrix
10489 inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
10490  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10491 {
10494  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10495 
10496  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
10497 
10500 
10501  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10502  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10503  }
10504 
10505  const AddType tmp( *this + (~rhs) );
10506 
10507  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10508  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10509  }
10510 
10511  decltype(auto) left( derestrict( *this ) );
10512 
10513  smpAssign( left, tmp );
10514 
10515  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10516 
10517  return *this;
10518 }
10520 //*************************************************************************************************
10521 
10522 
10523 //*************************************************************************************************
10537 template< typename MT // Type of the dense matrix
10538  , size_t... CSAs > // Compile time submatrix arguments
10539 template< typename MT2 // Type of the right-hand side matrix
10540  , bool SO > // Storage order of the right-hand side matrix
10541 inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
10542  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10543 {
10546  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10547 
10548  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
10549 
10552 
10553  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10554  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10555  }
10556 
10557  if( !trySubAssign( matrix_, ~rhs, row(), column() ) ) {
10558  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10559  }
10560 
10561  decltype(auto) left( derestrict( *this ) );
10562 
10563  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
10564  (~rhs).canAlias( &matrix_ ) ) {
10565  const SubType tmp( *this - (~rhs ) );
10566  smpAssign( left, tmp );
10567  }
10568  else {
10569  smpSubAssign( left, ~rhs );
10570  }
10571 
10572  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10573 
10574  return *this;
10575 }
10577 //*************************************************************************************************
10578 
10579 
10580 //*************************************************************************************************
10594 template< typename MT // Type of the dense matrix
10595  , size_t... CSAs > // Compile time submatrix arguments
10596 template< typename MT2 // Type of the right-hand side matrix
10597  , bool SO > // Storage order of the right-hand side matrix
10598 inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
10599  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10600 {
10603  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10604 
10605  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
10606 
10609 
10610  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10611  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10612  }
10613 
10614  const SubType tmp( *this - (~rhs) );
10615 
10616  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10617  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10618  }
10619 
10620  decltype(auto) left( derestrict( *this ) );
10621 
10622  smpAssign( left, tmp );
10623 
10624  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10625 
10626  return *this;
10627 }
10629 //*************************************************************************************************
10630 
10631 
10632 //*************************************************************************************************
10646 template< typename MT // Type of the dense matrix
10647  , size_t... CSAs > // Compile time submatrix arguments
10648 template< typename MT2 // Type of the right-hand side matrix
10649  , bool SO > // Storage order of the right-hand side matrix
10650 inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
10651  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10652 {
10655  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10656 
10657  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
10658 
10660 
10661  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10662  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10663  }
10664 
10665  if( !trySchurAssign( matrix_, ~rhs, row(), column() ) ) {
10666  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10667  }
10668 
10669  decltype(auto) left( derestrict( *this ) );
10670 
10671  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
10672  (~rhs).canAlias( &matrix_ ) ) {
10673  const SchurType tmp( *this % (~rhs) );
10674  if( IsSparseMatrix_v<SchurType> )
10675  reset();
10676  smpAssign( left, tmp );
10677  }
10678  else {
10679  smpSchurAssign( left, ~rhs );
10680  }
10681 
10682  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10683 
10684  return *this;
10685 }
10687 //*************************************************************************************************
10688 
10689 
10690 //*************************************************************************************************
10704 template< typename MT // Type of the dense matrix
10705  , size_t... CSAs > // Compile time submatrix arguments
10706 template< typename MT2 // Type of the right-hand side matrix
10707  , bool SO > // Storage order of the right-hand side matrix
10708 inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
10709  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10710 {
10713  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10714 
10715  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
10716 
10718 
10719  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10720  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10721  }
10722 
10723  const SchurType tmp( *this % (~rhs) );
10724 
10725  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10726  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10727  }
10728 
10729  decltype(auto) left( derestrict( *this ) );
10730 
10731  if( IsSparseMatrix_v<SchurType> ) {
10732  reset();
10733  }
10734 
10735  smpAssign( left, tmp );
10736 
10737  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10738 
10739  return *this;
10740 }
10742 //*************************************************************************************************
10743 
10744 
10745 
10746 
10747 //=================================================================================================
10748 //
10749 // UTILITY FUNCTIONS
10750 //
10751 //=================================================================================================
10752 
10753 //*************************************************************************************************
10759 template< typename MT // Type of the dense matrix
10760  , size_t... CSAs > // Compile time submatrix arguments
10761 inline MT& Submatrix<MT,aligned,true,true,CSAs...>::operand() noexcept
10762 {
10763  return matrix_;
10764 }
10766 //*************************************************************************************************
10767 
10768 
10769 //*************************************************************************************************
10775 template< typename MT // Type of the dense matrix
10776  , size_t... CSAs > // Compile time submatrix arguments
10777 inline const MT& Submatrix<MT,aligned,true,true,CSAs...>::operand() const noexcept
10778 {
10779  return matrix_;
10780 }
10782 //*************************************************************************************************
10783 
10784 
10785 //*************************************************************************************************
10794 template< typename MT // Type of the dense matrix
10795  , size_t... CSAs > // Compile time submatrix arguments
10796 inline size_t Submatrix<MT,aligned,true,true,CSAs...>::spacing() const noexcept
10797 {
10798  return matrix_.spacing();
10799 }
10801 //*************************************************************************************************
10802 
10803 
10804 //*************************************************************************************************
10810 template< typename MT // Type of the dense matrix
10811  , size_t... CSAs > // Compile time submatrix arguments
10812 inline size_t Submatrix<MT,aligned,true,true,CSAs...>::capacity() const noexcept
10813 {
10814  return rows() * columns();
10815 }
10817 //*************************************************************************************************
10818 
10819 
10820 //*************************************************************************************************
10827 template< typename MT // Type of the dense matrix
10828  , size_t... CSAs > // Compile time submatrix arguments
10829 inline size_t Submatrix<MT,aligned,true,true,CSAs...>::capacity( size_t j ) const noexcept
10830 {
10831  MAYBE_UNUSED( j );
10832 
10833  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
10834 
10835  return rows();
10836 }
10838 //*************************************************************************************************
10839 
10840 
10841 //*************************************************************************************************
10847 template< typename MT // Type of the dense matrix
10848  , size_t... CSAs > // Compile time submatrix arguments
10850 {
10851  const size_t iend( row() + rows() );
10852  const size_t jend( column() + columns() );
10853  size_t nonzeros( 0UL );
10854 
10855  for( size_t j=column(); j<jend; ++j )
10856  for( size_t i=row(); i<iend; ++i )
10857  if( !isDefault( matrix_(i,j) ) )
10858  ++nonzeros;
10859 
10860  return nonzeros;
10861 }
10863 //*************************************************************************************************
10864 
10865 
10866 //*************************************************************************************************
10873 template< typename MT // Type of the dense matrix
10874  , size_t... CSAs > // Compile time submatrix arguments
10875 inline size_t Submatrix<MT,aligned,true,true,CSAs...>::nonZeros( size_t j ) const
10876 {
10877  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
10878 
10879  const size_t iend( row() + rows() );
10880  size_t nonzeros( 0UL );
10881 
10882  for( size_t i=row(); i<iend; ++i )
10883  if( !isDefault( matrix_(i,column()+j) ) )
10884  ++nonzeros;
10885 
10886  return nonzeros;
10887 }
10889 //*************************************************************************************************
10890 
10891 
10892 //*************************************************************************************************
10898 template< typename MT // Type of the dense matrix
10899  , size_t... CSAs > // Compile time submatrix arguments
10901 {
10902  using blaze::clear;
10903 
10904  for( size_t j=column(); j<column()+columns(); ++j )
10905  {
10906  const size_t ibegin( ( IsLower_v<MT> )
10907  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
10908  ?( max( j+1UL, row() ) )
10909  :( max( j, row() ) ) )
10910  :( row() ) );
10911  const size_t iend ( ( IsUpper_v<MT> )
10912  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
10913  ?( min( j, row()+rows() ) )
10914  :( min( j+1UL, row()+rows() ) ) )
10915  :( row()+rows() ) );
10916 
10917  for( size_t i=ibegin; i<iend; ++i )
10918  clear( matrix_(i,j) );
10919  }
10920 }
10922 //*************************************************************************************************
10923 
10924 
10925 //*************************************************************************************************
10932 template< typename MT // Type of the dense matrix
10933  , size_t... CSAs > // Compile time submatrix arguments
10934 inline void Submatrix<MT,aligned,true,true,CSAs...>::reset( size_t j )
10935 {
10936  using blaze::clear;
10937 
10938  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
10939 
10940  const size_t ibegin( ( IsLower_v<MT> )
10941  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
10942  ?( max( j+1UL, row() ) )
10943  :( max( j, row() ) ) )
10944  :( row() ) );
10945  const size_t iend ( ( IsUpper_v<MT> )
10946  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
10947  ?( min( j, row()+rows() ) )
10948  :( min( j+1UL, row()+rows() ) ) )
10949  :( row()+rows() ) );
10950 
10951  for( size_t i=ibegin; i<iend; ++i )
10952  clear( matrix_(i,column()+j) );
10953 }
10955 //*************************************************************************************************
10956 
10957 
10958 //*************************************************************************************************
10968 template< typename MT // Type of the dense matrix
10969  , size_t... CSAs > // Compile time submatrix arguments
10970 inline bool Submatrix<MT,aligned,true,true,CSAs...>::hasOverlap() const noexcept
10971 {
10972  BLAZE_INTERNAL_ASSERT( IsSymmetric_v<MT> || IsHermitian_v<MT>, "Invalid matrix detected" );
10973 
10974  if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
10975  return false;
10976  else return true;
10977 }
10979 //*************************************************************************************************
10980 
10981 
10982 
10983 
10984 //=================================================================================================
10985 //
10986 // NUMERIC FUNCTIONS
10987 //
10988 //=================================================================================================
10989 
10990 //*************************************************************************************************
11008 template< typename MT // Type of the dense matrix
11009  , size_t... CSAs > // Compile time submatrix arguments
11010 inline Submatrix<MT,aligned,true,true,CSAs...>&
11012 {
11013  if( rows() != columns() ) {
11014  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
11015  }
11016 
11017  if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
11018  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
11019  }
11020 
11021  decltype(auto) left( derestrict( *this ) );
11022  const ResultType tmp( trans( *this ) );
11023 
11024  smpAssign( left, tmp );
11025 
11026  return *this;
11027 }
11029 //*************************************************************************************************
11030 
11031 
11032 //*************************************************************************************************
11050 template< typename MT // Type of the dense matrix
11051  , size_t... CSAs > // Compile time submatrix arguments
11052 inline Submatrix<MT,aligned,true,true,CSAs...>&
11053  Submatrix<MT,aligned,true,true,CSAs...>::ctranspose()
11054 {
11055  if( rows() != columns() ) {
11056  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
11057  }
11058 
11059  if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
11060  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
11061  }
11062 
11063  decltype(auto) left( derestrict( *this ) );
11064  const ResultType tmp( ctrans( *this ) );
11065 
11066  smpAssign( left, tmp );
11067 
11068  return *this;
11069 }
11071 //*************************************************************************************************
11072 
11073 
11074 //*************************************************************************************************
11087 template< typename MT // Type of the dense matrix
11088  , size_t... CSAs > // Compile time submatrix arguments
11089 template< typename Other > // Data type of the scalar value
11090 inline Submatrix<MT,aligned,true,true,CSAs...>&
11091  Submatrix<MT,aligned,true,true,CSAs...>::scale( const Other& scalar )
11092 {
11094 
11095  const size_t jend( column() + columns() );
11096 
11097  for( size_t j=column(); j<jend; ++j )
11098  {
11099  const size_t ibegin( ( IsLower_v<MT> )
11100  ?( ( IsStrictlyLower_v<MT> )
11101  ?( max( j+1UL, row() ) )
11102  :( max( j, row() ) ) )
11103  :( row() ) );
11104  const size_t iend ( ( IsUpper_v<MT> )
11105  ?( ( IsStrictlyUpper_v<MT> )
11106  ?( min( j, row()+rows() ) )
11107  :( min( j+1UL, row()+rows() ) ) )
11108  :( row()+rows() ) );
11109 
11110  for( size_t i=ibegin; i<iend; ++i )
11111  matrix_(i,j) *= scalar;
11112  }
11113 
11114  return *this;
11115 }
11117 //*************************************************************************************************
11118 
11119 
11120 
11121 
11122 //=================================================================================================
11123 //
11124 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
11125 //
11126 //=================================================================================================
11127 
11128 //*************************************************************************************************
11139 template< typename MT // Type of the dense matrix
11140  , size_t... CSAs > // Compile time submatrix arguments
11141 template< typename Other > // Data type of the foreign expression
11142 inline bool Submatrix<MT,aligned,true,true,CSAs...>::canAlias( const Other* alias ) const noexcept
11143 {
11144  return matrix_.isAliased( alias );
11145 }
11147 //*************************************************************************************************
11148 
11149 
11150 //*************************************************************************************************
11161 template< typename MT // Type of the dense matrix
11162  , size_t... CSAs > // Compile time submatrix arguments
11163 template< typename MT2 // Data type of the foreign dense submatrix
11164  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
11165  , bool SO2 // Storage order of the foreign dense submatrix
11166  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
11167 inline bool
11168  Submatrix<MT,aligned,true,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
11169 {
11170  return ( matrix_.isAliased( &alias->matrix_ ) &&
11171  ( row() + rows() > alias->row() ) &&
11172  ( row() < alias->row() + alias->rows() ) &&
11173  ( column() + columns() > alias->column() ) &&
11174  ( column() < alias->column() + alias->columns() ) );
11175 }
11177 //*************************************************************************************************
11178 
11179 
11180 //*************************************************************************************************
11191 template< typename MT // Type of the dense matrix
11192  , size_t... CSAs > // Compile time submatrix arguments
11193 template< typename Other > // Data type of the foreign expression
11194 inline bool Submatrix<MT,aligned,true,true,CSAs...>::isAliased( const Other* alias ) const noexcept
11195 {
11196  return matrix_.isAliased( alias );
11197 }
11199 //*************************************************************************************************
11200 
11201 
11202 //*************************************************************************************************
11213 template< typename MT // Type of the dense matrix
11214  , size_t... CSAs > // Compile time submatrix arguments
11215 template< typename MT2 // Data type of the foreign dense submatrix
11216  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
11217  , bool SO2 // Storage order of the foreign dense submatrix
11218  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
11219 inline bool
11220  Submatrix<MT,aligned,true,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
11221 {
11222  return ( matrix_.isAliased( &alias->matrix_ ) &&
11223  ( row() + rows() > alias->row() ) &&
11224  ( row() < alias->row() + alias->rows() ) &&
11225  ( column() + columns() > alias->column() ) &&
11226  ( column() < alias->column() + alias->columns() ) );
11227 }
11229 //*************************************************************************************************
11230 
11231 
11232 //*************************************************************************************************
11242 template< typename MT // Type of the dense matrix
11243  , size_t... CSAs > // Compile time submatrix arguments
11244 inline bool Submatrix<MT,aligned,true,true,CSAs...>::isAligned() const noexcept
11245 {
11246  return true;
11247 }
11249 //*************************************************************************************************
11250 
11251 
11252 //*************************************************************************************************
11263 template< typename MT // Type of the dense matrix
11264  , size_t... CSAs > // Compile time submatrix arguments
11265 inline bool Submatrix<MT,aligned,true,true,CSAs...>::canSMPAssign() const noexcept
11266 {
11267  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
11268 }
11270 //*************************************************************************************************
11271 
11272 
11273 //*************************************************************************************************
11288 template< typename MT // Type of the dense matrix
11289  , size_t... CSAs > // Compile time submatrix arguments
11290 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,true,true,CSAs...>::SIMDType
11291  Submatrix<MT,aligned,true,true,CSAs...>::load( size_t i, size_t j ) const noexcept
11292 {
11293  return loada( i, j );
11294 }
11296 //*************************************************************************************************
11297 
11298 
11299 //*************************************************************************************************
11314 template< typename MT // Type of the dense matrix
11315  , size_t... CSAs > // Compile time submatrix arguments
11316 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,true,true,CSAs...>::SIMDType
11317  Submatrix<MT,aligned,true,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
11318 {
11320 
11321  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11322  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11323  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11324  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11325 
11326  return matrix_.loada( row()+i, column()+j );
11327 }
11329 //*************************************************************************************************
11330 
11331 
11332 //*************************************************************************************************
11347 template< typename MT // Type of the dense matrix
11348  , size_t... CSAs > // Compile time submatrix arguments
11349 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,true,true,CSAs...>::SIMDType
11350  Submatrix<MT,aligned,true,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
11351 {
11353 
11354  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11355  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11356  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11357 
11358  return matrix_.loadu( row()+i, column()+j );
11359 }
11361 //*************************************************************************************************
11362 
11363 
11364 //*************************************************************************************************
11380 template< typename MT // Type of the dense matrix
11381  , size_t... CSAs > // Compile time submatrix arguments
11383  Submatrix<MT,aligned,true,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
11384 {
11385  storea( i, j, value );
11386 }
11388 //*************************************************************************************************
11389 
11390 
11391 //*************************************************************************************************
11407 template< typename MT // Type of the dense matrix
11408  , size_t... CSAs > // Compile time submatrix arguments
11410  Submatrix<MT,aligned,true,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
11411 {
11413 
11414  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11415  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11416  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11417  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11418 
11419  matrix_.storea( row()+i, column()+j, value );
11420 }
11422 //*************************************************************************************************
11423 
11424 
11425 //*************************************************************************************************
11441 template< typename MT // Type of the dense matrix
11442  , size_t... CSAs > // Compile time submatrix arguments
11444  Submatrix<MT,aligned,true,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
11445 {
11447 
11448  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11449  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11450  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11451 
11452  matrix_.storeu( row()+i, column()+j, value );
11453 }
11455 //*************************************************************************************************
11456 
11457 
11458 //*************************************************************************************************
11475 template< typename MT // Type of the dense matrix
11476  , size_t... CSAs > // Compile time submatrix arguments
11478  Submatrix<MT,aligned,true,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
11479 {
11481 
11482  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11483  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11484  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11485  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11486 
11487  matrix_.stream( row()+i, column()+j, value );
11488 }
11490 //*************************************************************************************************
11491 
11492 
11493 //*************************************************************************************************
11505 template< typename MT // Type of the dense matrix
11506  , size_t... CSAs > // Compile time submatrix arguments
11507 template< typename MT2 > // Type of the right-hand side dense matrix
11508 inline auto Submatrix<MT,aligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
11509  -> DisableIf_t< VectorizedAssign_v<MT2> >
11510 {
11511  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11512  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11513 
11514  const size_t ipos( rows() & size_t(-2) );
11515  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
11516 
11517  for( size_t j=0UL; j<columns(); ++j ) {
11518  for( size_t i=0UL; i<ipos; i+=2UL ) {
11519  matrix_(row()+i ,column()+j) = (~rhs)(i ,j);
11520  matrix_(row()+i+1UL,column()+j) = (~rhs)(i+1UL,j);
11521  }
11522  if( ipos < rows() ) {
11523  matrix_(row()+ipos,column()+j) = (~rhs)(ipos,j);
11524  }
11525  }
11526 }
11528 //*************************************************************************************************
11529 
11530 
11531 //*************************************************************************************************
11543 template< typename MT // Type of the dense matrix
11544  , size_t... CSAs > // Compile time submatrix arguments
11545 template< typename MT2 > // Type of the right-hand side dense matrix
11546 inline auto Submatrix<MT,aligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
11547  -> EnableIf_t< VectorizedAssign_v<MT2> >
11548 {
11550 
11551  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11552  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11553 
11554  const size_t ipos( rows() & size_t(-SIMDSIZE) );
11555  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
11556 
11557  if( useStreaming &&
11558  rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
11559  !(~rhs).isAliased( &matrix_ ) )
11560  {
11561  for( size_t j=0UL; j<columns(); ++j )
11562  {
11563  size_t i( 0UL );
11564  Iterator left( begin(j) );
11565  ConstIterator_t<MT2> right( (~rhs).begin(j) );
11566 
11567  for( ; i<ipos; i+=SIMDSIZE ) {
11568  left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11569  }
11570  for( ; i<rows(); ++i ) {
11571  *left = *right; ++left; ++right;
11572  }
11573  }
11574  }
11575  else
11576  {
11577  for( size_t j=0UL; j<columns(); ++j )
11578  {
11579  size_t i( 0UL );
11580  Iterator left( begin(j) );
11581  ConstIterator_t<MT2> right( (~rhs).begin(j) );
11582 
11583  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
11584  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11585  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11586  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11587  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11588  }
11589  for( ; i<ipos; i+=SIMDSIZE ) {
11590  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11591  }
11592  for( ; i<rows(); ++i ) {
11593  *left = *right; ++left; ++right;
11594  }
11595  }
11596  }
11597 }
11599 //*************************************************************************************************
11600 
11601 
11602 //*************************************************************************************************
11614 template< typename MT // Type of the dense matrix
11615  , size_t... CSAs > // Compile time submatrix arguments
11616 template< typename MT2 > // Type of the right-hand side dense matrix
11617 inline void Submatrix<MT,aligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
11618 {
11620 
11621  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11622  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11623 
11624  constexpr size_t block( BLOCK_SIZE );
11625 
11626  for( size_t jj=0UL; jj<columns(); jj+=block ) {
11627  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
11628  for( size_t ii=0UL; ii<rows(); ii+=block ) {
11629  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
11630  for( size_t j=jj; j<jend; ++j ) {
11631  for( size_t i=ii; i<iend; ++i ) {
11632  matrix_(row()+i,column()+j) = (~rhs)(i,j);
11633  }
11634  }
11635  }
11636  }
11637 }
11639 //*************************************************************************************************
11640 
11641 
11642 //*************************************************************************************************
11654 template< typename MT // Type of the dense matrix
11655  , size_t... CSAs > // Compile time submatrix arguments
11656 template< typename MT2 > // Type of the right-hand side sparse matrix
11657 inline void Submatrix<MT,aligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
11658 {
11659  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11660  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11661 
11662  for( size_t j=0UL; j<columns(); ++j )
11663  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
11664  matrix_(row()+element->index(),column()+j) = element->value();
11665 }
11667 //*************************************************************************************************
11668 
11669 
11670 //*************************************************************************************************
11682 template< typename MT // Type of the dense matrix
11683  , size_t... CSAs > // Compile time submatrix arguments
11684 template< typename MT2 > // Type of the right-hand side sparse matrix
11685 inline void Submatrix<MT,aligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
11686 {
11688 
11689  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11690  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11691 
11692  for( size_t i=0UL; i<rows(); ++i )
11693  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
11694  matrix_(row()+i,column()+element->index()) = element->value();
11695 }
11697 //*************************************************************************************************
11698 
11699 
11700 //*************************************************************************************************
11712 template< typename MT // Type of the dense matrix
11713  , size_t... CSAs > // Compile time submatrix arguments
11714 template< typename MT2 > // Type of the right-hand side dense matrix
11715 inline auto Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
11716  -> DisableIf_t< VectorizedAddAssign_v<MT2> >
11717 {
11718  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11719  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11720 
11721  const size_t ipos( rows() & size_t(-2) );
11722  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
11723 
11724  for( size_t j=0UL; j<columns(); ++j )
11725  {
11726  if( IsDiagonal_v<MT2> ) {
11727  matrix_(row()+j,column()+j) += (~rhs)(j,j);
11728  }
11729  else {
11730  for( size_t i=0UL; i<ipos; i+=2UL ) {
11731  matrix_(row()+i ,column()+j) += (~rhs)(i ,j);
11732  matrix_(row()+i+1UL,column()+j) += (~rhs)(i+1UL,j);
11733  }
11734  if( ipos < rows() ) {
11735  matrix_(row()+ipos,column()+j) += (~rhs)(ipos,j);
11736  }
11737  }
11738  }
11739 }
11741 //*************************************************************************************************
11742 
11743 
11744 //*************************************************************************************************
11756 template< typename MT // Type of the dense matrix
11757  , size_t... CSAs > // Compile time submatrix arguments
11758 template< typename MT2 > // Type of the right-hand side dense matrix
11759 inline auto Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
11760  -> EnableIf_t< VectorizedAddAssign_v<MT2> >
11761 {
11763 
11764  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11765  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11766 
11767  for( size_t j=0UL; j<columns(); ++j )
11768  {
11769  const size_t ibegin( ( IsLower_v<MT> )
11770  ?( ( IsStrictlyLower_v<MT> ? j+1UL : j ) & size_t(-SIMDSIZE) )
11771  :( 0UL ) );
11772  const size_t iend ( ( IsUpper_v<MT> )
11773  ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
11774  :( rows() ) );
11775  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
11776 
11777  const size_t ipos( iend & size_t(-SIMDSIZE) );
11778  BLAZE_INTERNAL_ASSERT( ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
11779 
11780  size_t i( ibegin );
11781  Iterator left( begin(j) + ibegin );
11782  ConstIterator_t<MT2> right( (~rhs).begin(j) + ibegin );
11783 
11784  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
11785  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11786  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11787  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11788  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11789  }
11790  for( ; i<ipos; i+=SIMDSIZE ) {
11791  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11792  }
11793  for( ; i<iend; ++i ) {
11794  *left += *right; ++left; ++right;
11795  }
11796  }
11797 }
11799 //*************************************************************************************************
11800 
11801 
11802 //*************************************************************************************************
11814 template< typename MT // Type of the dense matrix
11815  , size_t... CSAs > // Compile time submatrix arguments
11816 template< typename MT2 > // Type of the right-hand side dense matrix
11817 inline void Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
11818 {
11820 
11821  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11822  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11823 
11824  constexpr size_t block( BLOCK_SIZE );
11825 
11826  for( size_t jj=0UL; jj<columns(); jj+=block ) {
11827  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
11828  for( size_t ii=0UL; ii<rows(); ii+=block ) {
11829  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
11830  for( size_t j=jj; j<jend; ++j ) {
11831  for( size_t i=ii; i<iend; ++i ) {
11832  matrix_(row()+i,column()+j) += (~rhs)(i,j);
11833  }
11834  }
11835  }
11836  }
11837 }
11839 //*************************************************************************************************
11840 
11841 
11842 //*************************************************************************************************
11854 template< typename MT // Type of the dense matrix
11855  , size_t... CSAs > // Compile time submatrix arguments
11856 template< typename MT2 > // Type of the right-hand side sparse matrix
11857 inline void Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
11858 {
11859  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11860  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11861 
11862  for( size_t j=0UL; j<columns(); ++j )
11863  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
11864  matrix_(row()+element->index(),column()+j) += element->value();
11865 }
11867 //*************************************************************************************************
11868 
11869 
11870 //*************************************************************************************************
11882 template< typename MT // Type of the dense matrix
11883  , size_t... CSAs > // Compile time submatrix arguments
11884 template< typename MT2 > // Type of the right-hand side sparse matrix
11885 inline void Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
11886 {
11888 
11889  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11890  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11891 
11892  for( size_t i=0UL; i<rows(); ++i )
11893  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
11894  matrix_(row()+i,column()+element->index()) += element->value();
11895 }
11897 //*************************************************************************************************
11898 
11899 
11900 //*************************************************************************************************
11912 template< typename MT // Type of the dense matrix
11913  , size_t... CSAs > // Compile time submatrix arguments
11914 template< typename MT2 > // Type of the right-hand side dense matrix
11915 inline auto Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
11916  -> DisableIf_t< VectorizedSubAssign_v<MT2> >
11917 {
11918  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11919  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11920 
11921  const size_t ipos( rows() & size_t(-2) );
11922  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
11923 
11924  for( size_t j=0UL; j<columns(); ++j )
11925  {
11926  if( IsDiagonal_v<MT2> ) {
11927  matrix_(row()+j,column()+j) -= (~rhs)(j,j);
11928  }
11929  else {
11930  for( size_t i=0UL; i<ipos; i+=2UL ) {
11931  matrix_(row()+i ,column()+j) -= (~rhs)(i ,j);
11932  matrix_(row()+i+1UL,column()+j) -= (~rhs)(i+1UL,j);
11933  }
11934  if( ipos < rows() ) {
11935  matrix_(row()+ipos,column()+j) -= (~rhs)(ipos,j);
11936  }
11937  }
11938  }
11939 }
11941 //*************************************************************************************************
11942 
11943 
11944 //*************************************************************************************************
11956 template< typename MT // Type of the dense matrix
11957  , size_t... CSAs > // Compile time submatrix arguments
11958 template< typename MT2 > // Type of the right-hand side dense matrix
11959 inline auto Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
11960  -> EnableIf_t< VectorizedSubAssign_v<MT2> >
11961 {
11963 
11964  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11965  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11966 
11967  for( size_t j=0UL; j<columns(); ++j )
11968  {
11969  const size_t ibegin( ( IsLower_v<MT> )
11970  ?( ( IsStrictlyLower_v<MT> ? j+1UL : j ) & size_t(-SIMDSIZE) )
11971  :( 0UL ) );
11972  const size_t iend ( ( IsUpper_v<MT> )
11973  ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
11974  :( rows() ) );
11975  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
11976 
11977  const size_t ipos( iend & size_t(-SIMDSIZE) );
11978  BLAZE_INTERNAL_ASSERT( ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
11979 
11980  size_t i( ibegin );
11981  Iterator left( begin(j) + ibegin );
11982  ConstIterator_t<MT2> right( (~rhs).begin(j) + ibegin );
11983 
11984  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
11985  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11986  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11987  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11988  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11989  }
11990  for( ; i<ipos; i+=SIMDSIZE ) {
11991  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11992  }
11993  for( ; i<iend; ++i ) {
11994  *left -= *right; ++left; ++right;
11995  }
11996  }
11997 }
11999 //*************************************************************************************************
12000 
12001 
12002 //*************************************************************************************************
12014 template< typename MT // Type of the dense matrix
12015  , size_t... CSAs > // Compile time submatrix arguments
12016 template< typename MT2 > // Type of the right-hand side dense matrix
12017 inline void Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
12018 {
12020 
12021  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12022  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12023 
12024  constexpr size_t block( BLOCK_SIZE );
12025 
12026  for( size_t jj=0UL; jj<columns(); jj+=block ) {
12027  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
12028  for( size_t ii=0UL; ii<rows(); ii+=block ) {
12029  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
12030  for( size_t j=jj; j<jend; ++j ) {
12031  for( size_t i=ii; i<iend; ++i ) {
12032  matrix_(row()+i,column()+j) -= (~rhs)(i,j);
12033  }
12034  }
12035  }
12036  }
12037 }
12039 //*************************************************************************************************
12040 
12041 
12042 //*************************************************************************************************
12054 template< typename MT // Type of the dense matrix
12055  , size_t... CSAs > // Compile time submatrix arguments
12056 template< typename MT2 > // Type of the right-hand side sparse matrix
12057 inline void Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
12058 {
12059  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12060  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12061 
12062  for( size_t j=0UL; j<columns(); ++j )
12063  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
12064  matrix_(row()+element->index(),column()+j) -= element->value();
12065 }
12067 //*************************************************************************************************
12068 
12069 
12070 //*************************************************************************************************
12082 template< typename MT // Type of the dense matrix
12083  , size_t... CSAs > // Compile time submatrix arguments
12084 template< typename MT2 > // Type of the right-hand side sparse matrix
12085 inline void Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
12086 {
12088 
12089  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12090  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12091 
12092  for( size_t i=0UL; i<rows(); ++i )
12093  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
12094  matrix_(row()+i,column()+element->index()) -= element->value();
12095 }
12097 //*************************************************************************************************
12098 
12099 
12100 //*************************************************************************************************
12112 template< typename MT // Type of the dense matrix
12113  , size_t... CSAs > // Compile time submatrix arguments
12114 template< typename MT2 > // Type of the right-hand side dense matrix
12115 inline auto Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
12116  -> DisableIf_t< VectorizedSchurAssign_v<MT2> >
12117 {
12118  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12119  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12120 
12121  const size_t ipos( rows() & size_t(-2) );
12122  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
12123 
12124  for( size_t j=0UL; j<columns(); ++j ) {
12125  for( size_t i=0UL; i<ipos; i+=2UL ) {
12126  matrix_(row()+i ,column()+j) *= (~rhs)(i ,j);
12127  matrix_(row()+i+1UL,column()+j) *= (~rhs)(i+1UL,j);
12128  }
12129  if( ipos < rows() ) {
12130  matrix_(row()+ipos,column()+j) *= (~rhs)(ipos,j);
12131  }
12132  }
12133 }
12135 //*************************************************************************************************
12136 
12137 
12138 //*************************************************************************************************
12151 template< typename MT // Type of the dense matrix
12152  , size_t... CSAs > // Compile time submatrix arguments
12153 template< typename MT2 > // Type of the right-hand side dense matrix
12154 inline auto Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
12155  -> EnableIf_t< VectorizedSchurAssign_v<MT2> >
12156 {
12158 
12159  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12160  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12161 
12162  for( size_t j=0UL; j<columns(); ++j )
12163  {
12164  const size_t ipos( rows() & size_t(-SIMDSIZE) );
12165  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
12166 
12167  size_t i( 0UL );
12168  Iterator left( begin(j) );
12169  ConstIterator_t<MT2> right( (~rhs).begin(j) );
12170 
12171  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
12172  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12173  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12174  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12175  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12176  }
12177  for( ; i<ipos; i+=SIMDSIZE ) {
12178  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12179  }
12180  for( ; i<rows(); ++i ) {
12181  *left *= *right; ++left; ++right;
12182  }
12183  }
12184 }
12186 //*************************************************************************************************
12187 
12188 
12189 //*************************************************************************************************
12201 template< typename MT // Type of the dense matrix
12202  , size_t... CSAs > // Compile time submatrix arguments
12203 template< typename MT2 > // Type of the right-hand side dense matrix
12204 inline void Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
12205 {
12207 
12208  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12209  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12210 
12211  constexpr size_t block( BLOCK_SIZE );
12212 
12213  for( size_t jj=0UL; jj<columns(); jj+=block ) {
12214  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
12215  for( size_t ii=0UL; ii<rows(); ii+=block ) {
12216  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
12217  for( size_t j=jj; j<jend; ++j ) {
12218  for( size_t i=ii; i<iend; ++i ) {
12219  matrix_(row()+i,column()+j) *= (~rhs)(i,j);
12220  }
12221  }
12222  }
12223  }
12224 }
12226 //*************************************************************************************************
12227 
12228 
12229 //*************************************************************************************************
12241 template< typename MT // Type of the dense matrix
12242  , size_t... CSAs > // Compile time submatrix arguments
12243 template< typename MT2 > // Type of the right-hand side sparse matrix
12244 inline void Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
12245 {
12246  using blaze::reset;
12247 
12248  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12249  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12250 
12251  for( size_t j=0UL; j<columns(); ++j )
12252  {
12253  size_t i( 0UL );
12254 
12255  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element ) {
12256  for( ; i<element->index(); ++i )
12257  reset( matrix_(row()+i,column()+j) );
12258  matrix_(row()+i,column()+j) *= element->value();
12259  ++i;
12260  }
12261 
12262  for( ; i<rows(); ++i ) {
12263  reset( matrix_(row()+i,column()+j) );
12264  }
12265  }
12266 }
12268 //*************************************************************************************************
12269 
12270 
12271 //*************************************************************************************************
12283 template< typename MT // Type of the dense matrix
12284  , size_t... CSAs > // Compile time submatrix arguments
12285 template< typename MT2 > // Type of the right-hand side sparse matrix
12286 inline void Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
12287 {
12288  using blaze::reset;
12289 
12291 
12292  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12293  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12294 
12295  for( size_t i=0UL; i<rows(); ++i )
12296  {
12297  size_t j( 0UL );
12298 
12299  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element ) {
12300  for( ; j<element->index(); ++j )
12301  reset( matrix_(row()+i,column()+j) );
12302  matrix_(row()+i,column()+j) *= element->value();
12303  ++j;
12304  }
12305 
12306  for( ; j<columns(); ++j ) {
12307  reset( matrix_(row()+i,column()+j) );
12308  }
12309  }
12310 }
12312 //*************************************************************************************************
12313 
12314 } // namespace blaze
12315 
12316 #endif
AlignmentFlag
Alignment flag for (un-)aligned vectors and matrices.Via these flags it is possible to specify subvec...
Definition: AlignmentFlag.h:62
Constraint on the data type.
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Constraint on the data type.
Header file for auxiliary alias declarations.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:133
Headerfile for the generic min algorithm.
Header file for the blaze::checked and blaze::unchecked instances.
auto operator-=(DenseMatrix< MT, SO > &mat, ST scalar) -> EnableIf_t< IsNumeric_v< ST >, MT & >
Subtraction assignment operator for the subtraction of a dense matrix and a scalar value ( ).
Definition: DenseMatrix.h:432
Header file for kernel specific block sizes.
Header file for the Schur product trait.
#define BLAZE_USER_ASSERT(expr, msg)
Run time assertion macro for user checks.In case of an invalid run time expression,...
Definition: Assert.h:117
Header file for the alignment flag values.
Header file for the IsUniUpper type trait.
Header file for the subtraction trait.
size_t capacity(const Matrix< MT, SO > &matrix) noexcept
Returns the maximum capacity of the matrix.
Definition: Matrix.h:546
Header file for basic type definitions.
constexpr const DenseIterator< Type, AF > operator-(const DenseIterator< Type, AF > &it, ptrdiff_t inc) noexcept
Subtraction between a DenseIterator and an integral value.
Definition: DenseIterator.h:750
MT::ElementType * data(DenseMatrix< MT, SO > &dm) noexcept
Low-level data access to the dense matrix elements.
Definition: DenseMatrix.h:170
Header file for the View base class.
constexpr bool IsSIMDCombinable_v
Auxiliary variable template for the IsSIMDCombinable type trait.The IsSIMDCombinable_v variable templ...
Definition: IsSIMDCombinable.h:137
Header file for the IsSparseMatrix type trait.
Header file for the IsDiagonal type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_COMPUTATION_TYPE(T)
Constraint on the data type.In case the given data type T is a computational expression (i....
Definition: Computation.h:81
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type,...
Definition: DenseMatrix.h:61
constexpr bool operator<(const NegativeAccuracy< A > &lhs, const T &rhs)
Less-than comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:332
BLAZE_ALWAYS_INLINE const EnableIf_t< IsIntegral_v< T > &&HasSize_v< T, 1UL >, If_t< IsSigned_v< T >, SIMDint8, SIMDuint8 > > loadu(const T *address) noexcept
Loads a vector of 1-byte integral values.
Definition: Loadu.h:76
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:372
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:595
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SUBMATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a submatrix type (i.e....
Definition: Submatrix.h:81
void ctranspose(Matrix< MT, SO > &matrix)
In-place conjugate transpose of the given matrix.
Definition: Matrix.h:851
#define BLAZE_CONSTRAINT_MUST_NOT_BE_UNITRIANGULAR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a lower or upper unitriangular matrix ty...
Definition: UniTriangular.h:81
Header file for the MAYBE_UNUSED function template.
constexpr bool IsReference_v
Auxiliary variable template for the IsReference type trait.The IsReference_v variable template provid...
Definition: IsReference.h:95
size_t nonZeros(const Matrix< MT, SO > &matrix)
Returns the total number of non-zero elements in the matrix.
Definition: Matrix.h:584
Header file for the reset shim.
constexpr bool HasSIMDAdd_v
Auxiliary variable template for the HasSIMDAdd type trait.The HasSIMDAdd_v variable template provides...
Definition: HasSIMDAdd.h:187
Constraints on the storage order of matrix types.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_TRANSEXPR_TYPE(T)
Constraint on the data type.In case the given data type T is a transposition expression (i....
Definition: TransExpr.h:81
BLAZE_ALWAYS_INLINE EnableIf_t< IsIntegral_v< T1 > &&HasSize_v< T1, 1UL > > storeu(T1 *address, const SIMDi8< T2 > &value) noexcept
Unaligned store of a vector of 1-byte integral values.
Definition: Storeu.h:75
Header file for the RequiresEvaluation type trait.
constexpr bool IsHermitian_v
Auxiliary variable template for the IsHermitian type trait.The IsHermitian_v variable template provid...
Definition: IsHermitian.h:172
Header file for the extended initializer_list functionality.
System settings for performance optimizations.
Header file for the IsUniLower type trait.
typename SubmatrixTrait< MT, CSAs... >::Type SubmatrixTrait_t
Auxiliary alias declaration for the SubmatrixTrait type trait.The SubmatrixTrait_t alias declaration ...
Definition: SubmatrixTrait.h:171
constexpr size_t columns(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of columns of the matrix.
Definition: Matrix.h:514
MT::ConstIterator cend(const Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:482
MT::ConstIterator cbegin(const Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:416
typename T::ElementType ElementType_t
Alias declaration for nested ElementType type definitions.The ElementType_t alias declaration provide...
Definition: Aliases.h:170
Constraint on the data type.
size_t spacing(const DenseMatrix< MT, SO > &dm) noexcept
Returns the spacing between the beginning of two rows/columns.
Definition: DenseMatrix.h:253
Header file for the matrix storage order types.
Constraint on the data type.
Header file for the implementation of a matrix representation of an initializer list.
typename EnableIf< Condition, T >::Type EnableIf_t
Auxiliary type for the EnableIf class template.The EnableIf_t alias declaration provides a convenient...
Definition: EnableIf.h:138
constexpr bool IsContiguous_v
Auxiliary variable template for the IsContiguous type trait.The IsContiguous_v variable template prov...
Definition: IsContiguous.h:144
Headerfile for the generic max algorithm.
Header file for the DisableIf class template.
Header file for the IsStrictlyUpper type trait.
Header file for the IsSymmetric type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:58
#define BLAZE_ALWAYS_INLINE
Platform dependent setup of an enforced inline keyword.
Definition: Inline.h:85
decltype(auto) ctrans(const DenseMatrix< MT, SO > &dm)
Returns the conjugate transpose matrix of dm.
Definition: DMatMapExpr.h:1361
Header file for the If class template.
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
decltype(auto) operator *(const DenseMatrix< MT1, false > &lhs, const DenseMatrix< MT2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:9091
#define BLAZE_CONSTRAINT_MUST_NOT_BE_POINTER_TYPE(T)
Constraint on the data type.In case the given data type T is not a pointer type, a compilation error ...
Definition: Pointer.h:79
constexpr bool HasSIMDSub_v
Auxiliary variable template for the HasSIMDSub type trait.The HasSIMDSub_v variable template provides...
Definition: HasSIMDSub.h:187
decltype(auto) min(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise minimum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1162
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the implementation of the Submatrix base template.
Header file for the HasSIMDAdd type trait.
Header file for the DenseMatrix base class.
constexpr bool operator>(const NegativeAccuracy< A > &lhs, const T &rhs)
Greater-than comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:370
constexpr bool operator>=(const NegativeAccuracy< A > &, const T &rhs)
Greater-or-equal-than comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:446
auto operator+=(DenseMatrix< MT, SO > &mat, ST scalar) -> EnableIf_t< IsNumeric_v< ST >, MT & >
Addition assignment operator for the addition of a dense matrix and a scalar value ( ).
Definition: DenseMatrix.h:370
constexpr bool operator==(const NegativeAccuracy< A > &lhs, const T &rhs)
Equality comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:253
Header file for all SIMD functionality.
constexpr bool Contains_v
Auxiliary variable template for the Contains type trait.The Contains_v variable template provides a c...
Definition: Contains.h:138
Header file for the IsLower type trait.
constexpr void MAYBE_UNUSED(const Args &...)
Suppression of unused parameter warnings.
Definition: MaybeUnused.h:81
BLAZE_ALWAYS_INLINE EnableIf_t< IsIntegral_v< T1 > &&HasSize_v< T1, 1UL > > storea(T1 *address, const SIMDi8< T2 > &value) noexcept
Aligned store of a vector of 1-byte integral values.
Definition: Storea.h:78
Constraint on the data type.
Header file for the IsTriangular type trait.
#define BLAZE_CONSTRAINT_MUST_BE_VECTORIZABLE_TYPE(T)
Constraint on the data type.In case the given data type T is not a vectorizable data type,...
Definition: Vectorizable.h:61
Header file for the exception macros of the math module.
decltype(auto) max(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise maximum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1198
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:438
Header file for the implementation of the SubmatrixData class template.
constexpr bool operator!=(const NegativeAccuracy< A > &lhs, const T &rhs)
Inequality comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:293
Constraint on the data type.
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
void clear(const DiagonalProxy< MT > &proxy)
Clearing the represented element.
Definition: DiagonalProxy.h:615
Constraint on the data type.
Flag for aligned vectors and matrices.
Definition: AlignmentFlag.h:65
Header file for the IsSIMDCombinable type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type,...
Definition: Symmetric.h:79
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
Header file for the IsConst type trait.
Header file for run time assertion macros.
Header file for the addition trait.
auto smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:131
Header file for the Unique class template.
Header file for the submatrix trait.
Header file for the IsContiguous type trait.
Check< false > Unchecked
Type of the blaze::unchecked instance.blaze::Unchecked is the type of the blaze::unchecked instance,...
Definition: Check.h:96
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:133
Header file for the cache size of the target architecture.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type,...
Definition: Reference.h:79
Header file for the isDefault shim.
constexpr size_t size(const Matrix< MT, SO > &matrix) noexcept
Returns the total number of elements of the matrix.
Definition: Matrix.h:530
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:100
Constraint on the data type.
Constraint on the data type.
Header file for the HasSIMDSub type trait.
Constraints on the storage order of matrix types.
constexpr bool operator<=(const NegativeAccuracy< A > &, const T &rhs)
Less-or-equal-than comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:408
Header file for the HasMutableDataAccess type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
Header file for the IsReference type trait.
auto smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:194
constexpr size_t rows(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of rows of the matrix.
Definition: Matrix.h:498
constexpr const DenseIterator< Type, AF > operator+(const DenseIterator< Type, AF > &it, ptrdiff_t inc) noexcept
Addition between a DenseIterator and an integral value.
Definition: DenseIterator.h:718
BLAZE_ALWAYS_INLINE const EnableIf_t< IsIntegral_v< T > &&HasSize_v< T, 1UL >, If_t< IsSigned_v< T >, SIMDint8, SIMDuint8 > > loada(const T *address) noexcept
Loads a vector of 1-byte integral values.
Definition: Loada.h:79
BLAZE_ALWAYS_INLINE EnableIf_t< IsIntegral_v< T1 > &&HasSize_v< T1, 1UL > > stream(T1 *address, const SIMDi8< T2 > &value) noexcept
Aligned, non-temporal store of a vector of 1-byte integral values.
Definition: Stream.h:74
BLAZE_ALWAYS_INLINE bool checkAlignment(const T *address)
Checks the alignment of the given address.
Definition: AlignmentCheck.h:68
decltype(auto) trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:765
Flag for unaligned vectors and matrices.
Definition: AlignmentFlag.h:64
auto smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:162
constexpr bool HasSIMDMult_v
Auxiliary variable template for the HasSIMDMult type trait.The HasSIMDMult_v variable template provid...
Definition: HasSIMDMult.h:188
Header file for the alignment check function.
bool isIntact(const DiagonalMatrix< MT, SO, DF > &m)
Returns whether the invariants of the given diagonal matrix are intact.
Definition: DiagonalMatrix.h:264
constexpr bool IsSymmetric_v
Auxiliary variable template for the IsSymmetric type trait.The IsSymmetric_v variable template provid...
Definition: IsSymmetric.h:172
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:635
#define BLAZE_THROW_LOGIC_ERROR(MESSAGE)
Macro for the emission of a std::logic_error exception.This macro encapsulates the default way of Bla...
Definition: Exception.h:187
Header file for the IsUpper type trait.
typename DisableIf< Condition, T >::Type DisableIf_t
Auxiliary type for the DisableIf class template.The DisableIf_t alias declaration provides a convenie...
Definition: DisableIf.h:138
constexpr bool IsDiagonal_v
Auxiliary variable template for the IsDiagonal type trait.The IsDiagonal_v variable template provides...
Definition: IsDiagonal.h:148
Header file for the IsHermitian type trait.
Header file for the IsRestricted type trait.
System settings for the inline keywords.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression,...
Definition: Assert.h:101
constexpr bool IsSparseMatrix_v
Auxiliary variable template for the IsSparseMatrix type trait.The IsSparseMatrix_v variable template ...
Definition: IsSparseMatrix.h:138
Header file for the clear shim.
Header file for the IsExpression type trait class.
Constraint on the data type.
void transpose(Matrix< MT, SO > &matrix)
In-place transpose of the given matrix.
Definition: Matrix.h:825