Dense.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_VIEWS_SUBMATRIX_DENSE_H_
36 #define _BLAZE_MATH_VIEWS_SUBMATRIX_DENSE_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <algorithm>
44 #include <iterator>
45 #include <blaze/math/Aliases.h>
57 #include <blaze/math/Exception.h>
61 #include <blaze/math/shims/Clear.h>
63 #include <blaze/math/shims/Reset.h>
64 #include <blaze/math/SIMD.h>
90 #include <blaze/math/views/Check.h>
93 #include <blaze/system/Blocking.h>
94 #include <blaze/system/CacheSize.h>
95 #include <blaze/system/Inline.h>
101 #include <blaze/util/Assert.h>
105 #include <blaze/util/DisableIf.h>
106 #include <blaze/util/EnableIf.h>
107 #include <blaze/util/mpl/If.h>
108 #include <blaze/util/TypeList.h>
109 #include <blaze/util/Types.h>
112 #include <blaze/util/Unused.h>
113 
114 
115 namespace blaze {
116 
117 //=================================================================================================
118 //
119 // CLASS TEMPLATE SPECIALIZATION FOR UNALIGNED ROW-MAJOR DENSE SUBMATRICES
120 //
121 //=================================================================================================
122 
123 //*************************************************************************************************
131 template< typename MT // Type of the dense matrix
132  , size_t... CSAs > // Compile time submatrix arguments
133 class Submatrix<MT,unaligned,false,true,CSAs...>
134  : public View< DenseMatrix< Submatrix<MT,unaligned,false,true,CSAs...>, false > >
135  , private SubmatrixData<CSAs...>
136 {
137  private:
138  //**Type definitions****************************************************************************
139  using DataType = SubmatrixData<CSAs...>;
140  using Operand = If_t< IsExpression_v<MT>, MT, MT& >;
141  //**********************************************************************************************
142 
143  //**********************************************************************************************
145  template< typename MT1, typename MT2 >
146  static constexpr bool EnforceEvaluation_v =
147  ( IsRestricted_v<MT1> && RequiresEvaluation_v<MT2> );
148  //**********************************************************************************************
149 
150  public:
151  //**Type definitions****************************************************************************
153  using This = Submatrix<MT,unaligned,false,true,CSAs...>;
154 
155  using BaseType = DenseMatrix<This,false>;
156  using ViewedType = MT;
157  using ResultType = SubmatrixTrait_t<MT,CSAs...>;
158  using OppositeType = OppositeType_t<ResultType>;
159  using TransposeType = TransposeType_t<ResultType>;
160  using ElementType = ElementType_t<MT>;
161  using SIMDType = SIMDTrait_t<ElementType>;
162  using ReturnType = ReturnType_t<MT>;
163  using CompositeType = const Submatrix&;
164 
166  using ConstReference = ConstReference_t<MT>;
167 
169  using Reference = If_t< IsConst_v<MT>, ConstReference, Reference_t<MT> >;
170 
172  using ConstPointer = ConstPointer_t<MT>;
173 
175  using Pointer = If_t< IsConst_v<MT> || !HasMutableDataAccess_v<MT>, ConstPointer, Pointer_t<MT> >;
176  //**********************************************************************************************
177 
178  //**SubmatrixIterator class definition**********************************************************
181  template< typename IteratorType > // Type of the dense matrix iterator
182  class SubmatrixIterator
183  {
184  public:
185  //**Type definitions*************************************************************************
187  using IteratorCategory = typename std::iterator_traits<IteratorType>::iterator_category;
188 
190  using ValueType = typename std::iterator_traits<IteratorType>::value_type;
191 
193  using PointerType = typename std::iterator_traits<IteratorType>::pointer;
194 
196  using ReferenceType = typename std::iterator_traits<IteratorType>::reference;
197 
199  using DifferenceType = typename std::iterator_traits<IteratorType>::difference_type;
200 
201  // STL iterator requirements
202  using iterator_category = IteratorCategory;
203  using value_type = ValueType;
204  using pointer = PointerType;
205  using reference = ReferenceType;
206  using difference_type = DifferenceType;
207  //*******************************************************************************************
208 
209  //**Constructor******************************************************************************
212  inline SubmatrixIterator()
213  : iterator_ ( ) // Iterator to the current submatrix element
214  , isAligned_( false ) // Memory alignment flag
215  {}
216  //*******************************************************************************************
217 
218  //**Constructor******************************************************************************
224  inline SubmatrixIterator( IteratorType iterator, bool isMemoryAligned )
225  : iterator_ ( iterator ) // Iterator to the current submatrix element
226  , isAligned_( isMemoryAligned ) // Memory alignment flag
227  {}
228  //*******************************************************************************************
229 
230  //**Constructor******************************************************************************
235  template< typename IteratorType2 >
236  inline SubmatrixIterator( const SubmatrixIterator<IteratorType2>& it )
237  : iterator_ ( it.base() ) // Iterator to the current submatrix element
238  , isAligned_( it.isAligned() ) // Memory alignment flag
239  {}
240  //*******************************************************************************************
241 
242  //**Addition assignment operator*************************************************************
248  inline SubmatrixIterator& operator+=( size_t inc ) {
249  iterator_ += inc;
250  return *this;
251  }
252  //*******************************************************************************************
253 
254  //**Subtraction assignment operator**********************************************************
260  inline SubmatrixIterator& operator-=( size_t dec ) {
261  iterator_ -= dec;
262  return *this;
263  }
264  //*******************************************************************************************
265 
266  //**Prefix increment operator****************************************************************
271  inline SubmatrixIterator& operator++() {
272  ++iterator_;
273  return *this;
274  }
275  //*******************************************************************************************
276 
277  //**Postfix increment operator***************************************************************
282  inline const SubmatrixIterator operator++( int ) {
283  return SubmatrixIterator( iterator_++, isAligned_ );
284  }
285  //*******************************************************************************************
286 
287  //**Prefix decrement operator****************************************************************
292  inline SubmatrixIterator& operator--() {
293  --iterator_;
294  return *this;
295  }
296  //*******************************************************************************************
297 
298  //**Postfix decrement operator***************************************************************
303  inline const SubmatrixIterator operator--( int ) {
304  return SubmatrixIterator( iterator_--, isAligned_ );
305  }
306  //*******************************************************************************************
307 
308  //**Element access operator******************************************************************
313  inline ReferenceType operator*() const {
314  return *iterator_;
315  }
316  //*******************************************************************************************
317 
318  //**Element access operator******************************************************************
323  inline IteratorType operator->() const {
324  return iterator_;
325  }
326  //*******************************************************************************************
327 
328  //**Load function****************************************************************************
338  inline SIMDType load() const noexcept {
339  if( isAligned_ )
340  return loada();
341  else
342  return loadu();
343  }
344  //*******************************************************************************************
345 
346  //**Loada function***************************************************************************
356  inline SIMDType loada() const noexcept {
357  return iterator_.loada();
358  }
359  //*******************************************************************************************
360 
361  //**Loadu function***************************************************************************
371  inline SIMDType loadu() const noexcept {
372  return iterator_.loadu();
373  }
374  //*******************************************************************************************
375 
376  //**Store function***************************************************************************
387  inline void store( const SIMDType& value ) const {
388  if( isAligned_ ) {
389  storea( value );
390  }
391  else {
392  storeu( value );
393  }
394  }
395  //*******************************************************************************************
396 
397  //**Storea function**************************************************************************
408  inline void storea( const SIMDType& value ) const {
409  iterator_.storea( value );
410  }
411  //*******************************************************************************************
412 
413  //**Storeu function**************************************************************************
424  inline void storeu( const SIMDType& value ) const {
425  iterator_.storeu( value );
426  }
427  //*******************************************************************************************
428 
429  //**Stream function**************************************************************************
440  inline void stream( const SIMDType& value ) const {
441  iterator_.stream( value );
442  }
443  //*******************************************************************************************
444 
445  //**Equality operator************************************************************************
451  inline bool operator==( const SubmatrixIterator& rhs ) const {
452  return iterator_ == rhs.iterator_;
453  }
454  //*******************************************************************************************
455 
456  //**Inequality operator**********************************************************************
462  inline bool operator!=( const SubmatrixIterator& rhs ) const {
463  return iterator_ != rhs.iterator_;
464  }
465  //*******************************************************************************************
466 
467  //**Less-than operator***********************************************************************
473  inline bool operator<( const SubmatrixIterator& rhs ) const {
474  return iterator_ < rhs.iterator_;
475  }
476  //*******************************************************************************************
477 
478  //**Greater-than operator********************************************************************
484  inline bool operator>( const SubmatrixIterator& rhs ) const {
485  return iterator_ > rhs.iterator_;
486  }
487  //*******************************************************************************************
488 
489  //**Less-or-equal-than operator**************************************************************
495  inline bool operator<=( const SubmatrixIterator& rhs ) const {
496  return iterator_ <= rhs.iterator_;
497  }
498  //*******************************************************************************************
499 
500  //**Greater-or-equal-than operator***********************************************************
506  inline bool operator>=( const SubmatrixIterator& rhs ) const {
507  return iterator_ >= rhs.iterator_;
508  }
509  //*******************************************************************************************
510 
511  //**Subtraction operator*********************************************************************
517  inline DifferenceType operator-( const SubmatrixIterator& rhs ) const {
518  return iterator_ - rhs.iterator_;
519  }
520  //*******************************************************************************************
521 
522  //**Addition operator************************************************************************
529  friend inline const SubmatrixIterator operator+( const SubmatrixIterator& it, size_t inc ) {
530  return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
531  }
532  //*******************************************************************************************
533 
534  //**Addition operator************************************************************************
541  friend inline const SubmatrixIterator operator+( size_t inc, const SubmatrixIterator& it ) {
542  return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
543  }
544  //*******************************************************************************************
545 
546  //**Subtraction operator*********************************************************************
553  friend inline const SubmatrixIterator operator-( const SubmatrixIterator& it, size_t dec ) {
554  return SubmatrixIterator( it.iterator_ - dec, it.isAligned_ );
555  }
556  //*******************************************************************************************
557 
558  //**Base function****************************************************************************
563  inline IteratorType base() const {
564  return iterator_;
565  }
566  //*******************************************************************************************
567 
568  //**IsAligned function***********************************************************************
573  inline bool isAligned() const noexcept {
574  return isAligned_;
575  }
576  //*******************************************************************************************
577 
578  private:
579  //**Member variables*************************************************************************
580  IteratorType iterator_;
581  bool isAligned_;
582  //*******************************************************************************************
583  };
584  //**********************************************************************************************
585 
586  //**Type definitions****************************************************************************
588  using ConstIterator = SubmatrixIterator< ConstIterator_t<MT> >;
589 
591  using Iterator = If_t< IsConst_v<MT>, ConstIterator, SubmatrixIterator< Iterator_t<MT> > >;
592  //**********************************************************************************************
593 
594  //**Compilation flags***************************************************************************
596  static constexpr bool simdEnabled = MT::simdEnabled;
597 
599  static constexpr bool smpAssignable = MT::smpAssignable;
600  //**********************************************************************************************
601 
602  //**Constructors********************************************************************************
605  template< typename... RSAs >
606  explicit inline Submatrix( MT& matrix, RSAs... args );
607 
608  Submatrix( const Submatrix& ) = default;
610  //**********************************************************************************************
611 
612  //**Destructor**********************************************************************************
615  ~Submatrix() = default;
617  //**********************************************************************************************
618 
619  //**Data access functions***********************************************************************
622  inline Reference operator()( size_t i, size_t j );
623  inline ConstReference operator()( size_t i, size_t j ) const;
624  inline Reference at( size_t i, size_t j );
625  inline ConstReference at( size_t i, size_t j ) const;
626  inline Pointer data () noexcept;
627  inline ConstPointer data () const noexcept;
628  inline Pointer data ( size_t i ) noexcept;
629  inline ConstPointer data ( size_t i ) const noexcept;
630  inline Iterator begin ( size_t i );
631  inline ConstIterator begin ( size_t i ) const;
632  inline ConstIterator cbegin( size_t i ) const;
633  inline Iterator end ( size_t i );
634  inline ConstIterator end ( size_t i ) const;
635  inline ConstIterator cend ( size_t i ) const;
637  //**********************************************************************************************
638 
639  //**Assignment operators************************************************************************
642  inline Submatrix& operator=( const ElementType& rhs );
643  inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
644  inline Submatrix& operator=( const Submatrix& rhs );
645 
646  template< typename MT2, bool SO2 >
647  inline Submatrix& operator=( const Matrix<MT2,SO2>& rhs );
648 
649  template< typename MT2, bool SO2 >
650  inline auto operator+=( const Matrix<MT2,SO2>& rhs )
651  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
652 
653  template< typename MT2, bool SO2 >
654  inline auto operator+=( const Matrix<MT2,SO2>& rhs )
655  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
656 
657  template< typename MT2, bool SO2 >
658  inline auto operator-=( const Matrix<MT2,SO2>& rhs )
659  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
660 
661  template< typename MT2, bool SO2 >
662  inline auto operator-=( const Matrix<MT2,SO2>& rhs )
663  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
664 
665  template< typename MT2, bool SO2 >
666  inline auto operator%=( const Matrix<MT2,SO2>& rhs )
667  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
668 
669  template< typename MT2, bool SO2 >
670  inline auto operator%=( const Matrix<MT2,SO2>& rhs )
671  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
673  //**********************************************************************************************
674 
675  //**Utility functions***************************************************************************
678  using DataType::row;
679  using DataType::column;
680  using DataType::rows;
681  using DataType::columns;
682 
683  inline MT& operand() noexcept;
684  inline const MT& operand() const noexcept;
685 
686  inline size_t spacing() const noexcept;
687  inline size_t capacity() const noexcept;
688  inline size_t capacity( size_t i ) const noexcept;
689  inline size_t nonZeros() const;
690  inline size_t nonZeros( size_t i ) const;
691  inline void reset();
692  inline void reset( size_t i );
694  //**********************************************************************************************
695 
696  //**Numeric functions***************************************************************************
699  inline Submatrix& transpose();
700  inline Submatrix& ctranspose();
701 
702  template< typename Other > inline Submatrix& scale( const Other& scalar );
704  //**********************************************************************************************
705 
706  private:
707  //**********************************************************************************************
709  template< typename MT2 >
710  static constexpr bool VectorizedAssign_v =
711  ( useOptimizedKernels &&
712  simdEnabled && MT2::simdEnabled &&
714  //**********************************************************************************************
715 
716  //**********************************************************************************************
718  template< typename MT2 >
719  static constexpr bool VectorizedAddAssign_v =
720  ( useOptimizedKernels &&
721  simdEnabled && MT2::simdEnabled &&
724  !IsDiagonal_v<MT2> );
725  //**********************************************************************************************
726 
727  //**********************************************************************************************
729  template< typename MT2 >
730  static constexpr bool VectorizedSubAssign_v =
731  ( useOptimizedKernels &&
732  simdEnabled && MT2::simdEnabled &&
735  !IsDiagonal_v<MT2> );
736  //**********************************************************************************************
737 
738  //**********************************************************************************************
740  template< typename MT2 >
741  static constexpr bool VectorizedSchurAssign_v =
742  ( useOptimizedKernels &&
743  simdEnabled && MT2::simdEnabled &&
746  //**********************************************************************************************
747 
748  //**SIMD properties*****************************************************************************
750  static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
751  //**********************************************************************************************
752 
753  public:
754  //**Expression template evaluation functions****************************************************
757  template< typename Other >
758  inline bool canAlias( const Other* alias ) const noexcept;
759 
760  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
761  inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
762 
763  template< typename Other >
764  inline bool isAliased( const Other* alias ) const noexcept;
765 
766  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
767  inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
768 
769  inline bool isAligned () const noexcept;
770  inline bool canSMPAssign() const noexcept;
771 
772  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
773  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
774  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
775 
776  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
777  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
778  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
779  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
780 
781  template< typename MT2 >
782  inline auto assign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT2> >;
783 
784  template< typename MT2 >
785  inline auto assign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT2> >;
786 
787  template< typename MT2 > inline void assign( const DenseMatrix<MT2,true>& rhs );
788  template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
789  template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
790 
791  template< typename MT2 >
792  inline auto addAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT2> >;
793 
794  template< typename MT2 >
795  inline auto addAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT2> >;
796 
797  template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,true>& rhs );
798  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
799  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
800 
801  template< typename MT2 >
802  inline auto subAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT2> >;
803 
804  template< typename MT2 >
805  inline auto subAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT2> >;
806 
807  template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,true>& rhs );
808  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
809  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
810 
811  template< typename MT2 >
812  inline auto schurAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT2> >;
813 
814  template< typename MT2 >
815  inline auto schurAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT2> >;
816 
817  template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,true>& rhs );
818  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
819  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
821  //**********************************************************************************************
822 
823  private:
824  //**Utility functions***************************************************************************
827  inline bool hasOverlap() const noexcept;
829  //**********************************************************************************************
830 
831  //**Member variables****************************************************************************
834  Operand matrix_;
835  const bool isAligned_;
836 
843  //**********************************************************************************************
844 
845  //**Friend declarations*************************************************************************
846  template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
847  //**********************************************************************************************
848 
849  //**Compile time checks*************************************************************************
857  //**********************************************************************************************
858 };
860 //*************************************************************************************************
861 
862 
863 
864 
865 //=================================================================================================
866 //
867 // CONSTRUCTORS
868 //
869 //=================================================================================================
870 
871 //*************************************************************************************************
884 template< typename MT // Type of the dense matrix
885  , size_t... CSAs > // Compile time submatrix arguments
886 template< typename... RSAs > // Runtime submatrix arguments
887 inline Submatrix<MT,unaligned,false,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
888  : DataType ( args... ) // Base class initialization
889  , matrix_ ( matrix ) // The matrix containing the submatrix
890  , isAligned_( simdEnabled && IsContiguous_v<MT> &&
891  matrix.data() != nullptr && checkAlignment( data() ) &&
892  ( rows() < 2UL || ( matrix.spacing() & size_t(-SIMDSIZE) ) == 0UL ) )
893 {
894  if( !Contains_v< TypeList<RSAs...>, Unchecked > ) {
895  if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
896  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
897  }
898  }
899  else {
900  BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
901  BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
902  }
903 }
905 //*************************************************************************************************
906 
907 
908 
909 
910 //=================================================================================================
911 //
912 // DATA ACCESS FUNCTIONS
913 //
914 //=================================================================================================
915 
916 //*************************************************************************************************
927 template< typename MT // Type of the dense matrix
928  , size_t... CSAs > // Compile time submatrix arguments
929 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Reference
930  Submatrix<MT,unaligned,false,true,CSAs...>::operator()( size_t i, size_t j )
931 {
932  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
933  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
934 
935  return matrix_(row()+i,column()+j);
936 }
938 //*************************************************************************************************
939 
940 
941 //*************************************************************************************************
952 template< typename MT // Type of the dense matrix
953  , size_t... CSAs > // Compile time submatrix arguments
954 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstReference
955  Submatrix<MT,unaligned,false,true,CSAs...>::operator()( size_t i, size_t j ) const
956 {
957  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
958  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
959 
960  return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
961 }
963 //*************************************************************************************************
964 
965 
966 //*************************************************************************************************
978 template< typename MT // Type of the dense matrix
979  , size_t... CSAs > // Compile time submatrix arguments
980 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Reference
981  Submatrix<MT,unaligned,false,true,CSAs...>::at( size_t i, size_t j )
982 {
983  if( i >= rows() ) {
984  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
985  }
986  if( j >= columns() ) {
987  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
988  }
989  return (*this)(i,j);
990 }
992 //*************************************************************************************************
993 
994 
995 //*************************************************************************************************
1007 template< typename MT // Type of the dense matrix
1008  , size_t... CSAs > // Compile time submatrix arguments
1009 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstReference
1010  Submatrix<MT,unaligned,false,true,CSAs...>::at( size_t i, size_t j ) const
1011 {
1012  if( i >= rows() ) {
1013  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
1014  }
1015  if( j >= columns() ) {
1016  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
1017  }
1018  return (*this)(i,j);
1019 }
1021 //*************************************************************************************************
1022 
1023 
1024 //*************************************************************************************************
1034 template< typename MT // Type of the dense matrix
1035  , size_t... CSAs > // Compile time submatrix arguments
1036 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Pointer
1038 {
1039  return matrix_.data() + row()*spacing() + column();
1040 }
1042 //*************************************************************************************************
1043 
1044 
1045 //*************************************************************************************************
1055 template< typename MT // Type of the dense matrix
1056  , size_t... CSAs > // Compile time submatrix arguments
1057 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstPointer
1059 {
1060  return matrix_.data() + row()*spacing() + column();
1061 }
1063 //*************************************************************************************************
1064 
1065 
1066 //*************************************************************************************************
1075 template< typename MT // Type of the dense matrix
1076  , size_t... CSAs > // Compile time submatrix arguments
1077 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Pointer
1079 {
1080  return matrix_.data() + (row()+i)*spacing() + column();
1081 }
1083 //*************************************************************************************************
1084 
1085 
1086 //*************************************************************************************************
1095 template< typename MT // Type of the dense matrix
1096  , size_t... CSAs > // Compile time submatrix arguments
1097 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstPointer
1098  Submatrix<MT,unaligned,false,true,CSAs...>::data( size_t i ) const noexcept
1099 {
1100  return matrix_.data() + (row()+i)*spacing() + column();
1101 }
1103 //*************************************************************************************************
1104 
1105 
1106 //*************************************************************************************************
1118 template< typename MT // Type of the dense matrix
1119  , size_t... CSAs > // Compile time submatrix arguments
1120 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Iterator
1122 {
1123  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1124  return Iterator( matrix_.begin( row() + i ) + column(), isAligned_ );
1125 }
1127 //*************************************************************************************************
1128 
1129 
1130 //*************************************************************************************************
1142 template< typename MT // Type of the dense matrix
1143  , size_t... CSAs > // Compile time submatrix arguments
1144 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1146 {
1147  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1148  return ConstIterator( matrix_.cbegin( row() + i ) + column(), isAligned_ );
1149 }
1151 //*************************************************************************************************
1152 
1153 
1154 //*************************************************************************************************
1166 template< typename MT // Type of the dense matrix
1167  , size_t... CSAs > // Compile time submatrix arguments
1168 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1170 {
1171  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1172  return ConstIterator( matrix_.cbegin( row() + i ) + column(), isAligned_ );
1173 }
1175 //*************************************************************************************************
1176 
1177 
1178 //*************************************************************************************************
1190 template< typename MT // Type of the dense matrix
1191  , size_t... CSAs > // Compile time submatrix arguments
1192 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::Iterator
1194 {
1195  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1196  return Iterator( matrix_.begin( row() + i ) + column() + columns(), isAligned_ );
1197 }
1199 //*************************************************************************************************
1200 
1201 
1202 //*************************************************************************************************
1214 template< typename MT // Type of the dense matrix
1215  , size_t... CSAs > // Compile time submatrix arguments
1216 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1218 {
1219  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1220  return ConstIterator( matrix_.cbegin( row() + i ) + column() + columns(), isAligned_ );
1221 }
1223 //*************************************************************************************************
1224 
1225 
1226 //*************************************************************************************************
1238 template< typename MT // Type of the dense matrix
1239  , size_t... CSAs > // Compile time submatrix arguments
1240 inline typename Submatrix<MT,unaligned,false,true,CSAs...>::ConstIterator
1242 {
1243  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
1244  return ConstIterator( matrix_.cbegin( row() + i ) + column() + columns(), isAligned_ );
1245 }
1247 //*************************************************************************************************
1248 
1249 
1250 
1251 
1252 //=================================================================================================
1253 //
1254 // ASSIGNMENT OPERATORS
1255 //
1256 //=================================================================================================
1257 
1258 //*************************************************************************************************
1269 template< typename MT // Type of the dense matrix
1270  , size_t... CSAs > // Compile time submatrix arguments
1271 inline Submatrix<MT,unaligned,false,true,CSAs...>&
1272  Submatrix<MT,unaligned,false,true,CSAs...>::operator=( const ElementType& rhs )
1273 {
1274  const size_t iend( row() + rows() );
1275  decltype(auto) left( derestrict( matrix_ ) );
1276 
1277  for( size_t i=row(); i<iend; ++i )
1278  {
1279  const size_t jbegin( ( IsUpper_v<MT> )
1280  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
1281  ?( max( i+1UL, column() ) )
1282  :( max( i, column() ) ) )
1283  :( column() ) );
1284  const size_t jend ( ( IsLower_v<MT> )
1285  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
1286  ?( min( i, column()+columns() ) )
1287  :( min( i+1UL, column()+columns() ) ) )
1288  :( column()+columns() ) );
1289 
1290  for( size_t j=jbegin; j<jend; ++j ) {
1291  if( !IsRestricted_v<MT> || IsTriangular_v<MT> || trySet( matrix_, i, j, rhs ) )
1292  left(i,j) = rhs;
1293  }
1294  }
1295 
1296  return *this;
1297 }
1299 //*************************************************************************************************
1300 
1301 
1302 //*************************************************************************************************
1318 template< typename MT // Type of the dense matrix
1319  , size_t... CSAs > // Compile time submatrix arguments
1320 inline Submatrix<MT,unaligned,false,true,CSAs...>&
1321  Submatrix<MT,unaligned,false,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
1322 {
1323  if( list.size() != rows() ) {
1324  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
1325  }
1326 
1327  if( IsRestricted_v<MT> ) {
1328  const InitializerMatrix<ElementType> tmp( list, columns() );
1329  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1330  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1331  }
1332  }
1333 
1334  decltype(auto) left( derestrict( *this ) );
1335  size_t i( 0UL );
1336 
1337  for( const auto& rowList : list ) {
1338  std::fill( std::copy( rowList.begin(), rowList.end(), left.begin(i) ), left.end(i), ElementType() );
1339  ++i;
1340  }
1341 
1342  return *this;
1343 }
1345 //*************************************************************************************************
1346 
1347 
1348 //*************************************************************************************************
1363 template< typename MT // Type of the dense matrix
1364  , size_t... CSAs > // Compile time submatrix arguments
1365 inline Submatrix<MT,unaligned,false,true,CSAs...>&
1366  Submatrix<MT,unaligned,false,true,CSAs...>::operator=( const Submatrix& rhs )
1367 {
1370 
1371  if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
1372  return *this;
1373 
1374  if( rows() != rhs.rows() || columns() != rhs.columns() ) {
1375  BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
1376  }
1377 
1378  if( !tryAssign( matrix_, rhs, row(), column() ) ) {
1379  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1380  }
1381 
1382  decltype(auto) left( derestrict( *this ) );
1383 
1384  if( rhs.canAlias( &matrix_ ) ) {
1385  const ResultType tmp( rhs );
1386  smpAssign( left, tmp );
1387  }
1388  else {
1389  smpAssign( left, rhs );
1390  }
1391 
1392  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1393 
1394  return *this;
1395 }
1397 //*************************************************************************************************
1398 
1399 
1400 //*************************************************************************************************
1415 template< typename MT // Type of the dense matrix
1416  , size_t... CSAs > // Compile time submatrix arguments
1417 template< typename MT2 // Type of the right-hand side matrix
1418  , bool SO2 > // Storage order of the right-hand side matrix
1419 inline Submatrix<MT,unaligned,false,true,CSAs...>&
1420  Submatrix<MT,unaligned,false,true,CSAs...>::operator=( const Matrix<MT2,SO2>& rhs )
1421 {
1422  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1423 
1424  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1425  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1426  }
1427 
1428  using Right = If_t< IsRestricted_v<MT>, CompositeType_t<MT2>, const MT2& >;
1429  Right right( ~rhs );
1430 
1431  if( !tryAssign( matrix_, right, row(), column() ) ) {
1432  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1433  }
1434 
1435  decltype(auto) left( derestrict( *this ) );
1436 
1437  if( IsReference_v<Right> && right.canAlias( &matrix_ ) ) {
1438  const ResultType_t<MT2> tmp( right );
1439  if( IsSparseMatrix_v<MT2> )
1440  reset();
1441  smpAssign( left, tmp );
1442  }
1443  else {
1444  if( IsSparseMatrix_v<MT2> )
1445  reset();
1446  smpAssign( left, right );
1447  }
1448 
1449  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1450 
1451  return *this;
1452 }
1454 //*************************************************************************************************
1455 
1456 
1457 //*************************************************************************************************
1471 template< typename MT // Type of the dense matrix
1472  , size_t... CSAs > // Compile time submatrix arguments
1473 template< typename MT2 // Type of the right-hand side matrix
1474  , bool SO2 > // Storage order of the right-hand side matrix
1475 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO2>& rhs )
1476  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1477 {
1480  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1481 
1482  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
1483 
1486 
1487  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1488  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1489  }
1490 
1491  if( !tryAddAssign( matrix_, ~rhs, row(), column() ) ) {
1492  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1493  }
1494 
1495  decltype(auto) left( derestrict( *this ) );
1496 
1497  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
1498  (~rhs).canAlias( &matrix_ ) ) {
1499  const AddType tmp( *this + (~rhs) );
1500  smpAssign( left, tmp );
1501  }
1502  else {
1503  smpAddAssign( left, ~rhs );
1504  }
1505 
1506  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1507 
1508  return *this;
1509 }
1511 //*************************************************************************************************
1512 
1513 
1514 //*************************************************************************************************
1528 template< typename MT // Type of the dense matrix
1529  , size_t... CSAs > // Compile time submatrix arguments
1530 template< typename MT2 // Type of the right-hand side matrix
1531  , bool SO2 > // Storage order of the right-hand side matrix
1532 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO2>& rhs )
1533  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1534 {
1537  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1538 
1539  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
1540 
1543 
1544  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1545  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1546  }
1547 
1548  const AddType tmp( *this + (~rhs) );
1549 
1550  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1551  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1552  }
1553 
1554  decltype(auto) left( derestrict( *this ) );
1555 
1556  smpAssign( left, tmp );
1557 
1558  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1559 
1560  return *this;
1561 }
1563 //*************************************************************************************************
1564 
1565 
1566 //*************************************************************************************************
1580 template< typename MT // Type of the dense matrix
1581  , size_t... CSAs > // Compile time submatrix arguments
1582 template< typename MT2 // Type of the right-hand side matrix
1583  , bool SO2 > // Storage order of the right-hand side matrix
1584 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO2>& rhs )
1585  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1586 {
1589  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1590 
1591  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
1592 
1595 
1596  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1597  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1598  }
1599 
1600  if( !trySubAssign( matrix_, ~rhs, row(), column() ) ) {
1601  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1602  }
1603 
1604  decltype(auto) left( derestrict( *this ) );
1605 
1606  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
1607  (~rhs).canAlias( &matrix_ ) ) {
1608  const SubType tmp( *this - (~rhs ) );
1609  smpAssign( left, tmp );
1610  }
1611  else {
1612  smpSubAssign( left, ~rhs );
1613  }
1614 
1615  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1616 
1617  return *this;
1618 }
1620 //*************************************************************************************************
1621 
1622 
1623 //*************************************************************************************************
1637 template< typename MT // Type of the dense matrix
1638  , size_t... CSAs > // Compile time submatrix arguments
1639 template< typename MT2 // Type of the right-hand side matrix
1640  , bool SO2 > // Storage order of the right-hand side matrix
1641 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO2>& rhs )
1642  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1643 {
1646  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1647 
1648  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
1649 
1652 
1653  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1654  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1655  }
1656 
1657  const SubType tmp( *this - (~rhs) );
1658 
1659  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1660  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1661  }
1662 
1663  decltype(auto) left( derestrict( *this ) );
1664 
1665  smpAssign( left, tmp );
1666 
1667  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1668 
1669  return *this;
1670 }
1672 //*************************************************************************************************
1673 
1674 
1675 //*************************************************************************************************
1689 template< typename MT // Type of the dense matrix
1690  , size_t... CSAs > // Compile time submatrix arguments
1691 template< typename MT2 // Type of the right-hand side matrix
1692  , bool SO2 > // Storage order of the right-hand side matrix
1693 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO2>& rhs )
1694  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1695 {
1698  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1699 
1700  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
1701 
1703 
1704  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1705  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1706  }
1707 
1708  if( !trySchurAssign( matrix_, ~rhs, row(), column() ) ) {
1709  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1710  }
1711 
1712  decltype(auto) left( derestrict( *this ) );
1713 
1714  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
1715  (~rhs).canAlias( &matrix_ ) ) {
1716  const SchurType tmp( *this % (~rhs) );
1717  if( IsSparseMatrix_v<SchurType> )
1718  reset();
1719  smpAssign( left, tmp );
1720  }
1721  else {
1722  smpSchurAssign( left, ~rhs );
1723  }
1724 
1725  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1726 
1727  return *this;
1728 }
1730 //*************************************************************************************************
1731 
1732 
1733 //*************************************************************************************************
1747 template< typename MT // Type of the dense matrix
1748  , size_t... CSAs > // Compile time submatrix arguments
1749 template< typename MT2 // Type of the right-hand side matrix
1750  , bool SO2 > // Storage order of the right-hand side matrix
1751 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO2>& rhs )
1752  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
1753 {
1756  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
1757 
1758  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
1759 
1761 
1762  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
1763  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
1764  }
1765 
1766  const SchurType tmp( *this % (~rhs) );
1767 
1768  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
1769  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
1770  }
1771 
1772  decltype(auto) left( derestrict( *this ) );
1773 
1774  if( IsSparseMatrix_v<SchurType> ) {
1775  reset();
1776  }
1777 
1778  smpAssign( left, tmp );
1779 
1780  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
1781 
1782  return *this;
1783 }
1785 //*************************************************************************************************
1786 
1787 
1788 
1789 
1790 //=================================================================================================
1791 //
1792 // UTILITY FUNCTIONS
1793 //
1794 //=================================================================================================
1795 
1796 //*************************************************************************************************
1802 template< typename MT // Type of the dense matrix
1803  , size_t... CSAs > // Compile time submatrix arguments
1804 inline MT& Submatrix<MT,unaligned,false,true,CSAs...>::operand() noexcept
1805 {
1806  return matrix_;
1807 }
1809 //*************************************************************************************************
1810 
1811 
1812 //*************************************************************************************************
1818 template< typename MT // Type of the dense matrix
1819  , size_t... CSAs > // Compile time submatrix arguments
1820 inline const MT& Submatrix<MT,unaligned,false,true,CSAs...>::operand() const noexcept
1821 {
1822  return matrix_;
1823 }
1825 //*************************************************************************************************
1826 
1827 
1828 //*************************************************************************************************
1839 template< typename MT // Type of the dense matrix
1840  , size_t... CSAs > // Compile time submatrix arguments
1841 inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::spacing() const noexcept
1842 {
1843  return matrix_.spacing();
1844 }
1846 //*************************************************************************************************
1847 
1848 
1849 //*************************************************************************************************
1855 template< typename MT // Type of the dense matrix
1856  , size_t... CSAs > // Compile time submatrix arguments
1857 inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::capacity() const noexcept
1858 {
1859  return rows() * columns();
1860 }
1862 //*************************************************************************************************
1863 
1864 
1865 //*************************************************************************************************
1877 template< typename MT // Type of the dense matrix
1878  , size_t... CSAs > // Compile time submatrix arguments
1879 inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::capacity( size_t i ) const noexcept
1880 {
1881  UNUSED_PARAMETER( i );
1882 
1883  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1884 
1885  return columns();
1886 }
1888 //*************************************************************************************************
1889 
1890 
1891 //*************************************************************************************************
1897 template< typename MT // Type of the dense matrix
1898  , size_t... CSAs > // Compile time submatrix arguments
1900 {
1901  const size_t iend( row() + rows() );
1902  const size_t jend( column() + columns() );
1903  size_t nonzeros( 0UL );
1904 
1905  for( size_t i=row(); i<iend; ++i )
1906  for( size_t j=column(); j<jend; ++j )
1907  if( !isDefault( matrix_(i,j) ) )
1908  ++nonzeros;
1909 
1910  return nonzeros;
1911 }
1913 //*************************************************************************************************
1914 
1915 
1916 //*************************************************************************************************
1928 template< typename MT // Type of the dense matrix
1929  , size_t... CSAs > // Compile time submatrix arguments
1930 inline size_t Submatrix<MT,unaligned,false,true,CSAs...>::nonZeros( size_t i ) const
1931 {
1932  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1933 
1934  const size_t jend( column() + columns() );
1935  size_t nonzeros( 0UL );
1936 
1937  for( size_t j=column(); j<jend; ++j )
1938  if( !isDefault( matrix_(row()+i,j) ) )
1939  ++nonzeros;
1940 
1941  return nonzeros;
1942 }
1944 //*************************************************************************************************
1945 
1946 
1947 //*************************************************************************************************
1953 template< typename MT // Type of the dense matrix
1954  , size_t... CSAs > // Compile time submatrix arguments
1956 {
1957  using blaze::clear;
1958 
1959  for( size_t i=row(); i<row()+rows(); ++i )
1960  {
1961  const size_t jbegin( ( IsUpper_v<MT> )
1962  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
1963  ?( max( i+1UL, column() ) )
1964  :( max( i, column() ) ) )
1965  :( column() ) );
1966  const size_t jend ( ( IsLower_v<MT> )
1967  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
1968  ?( min( i, column()+columns() ) )
1969  :( min( i+1UL, column()+columns() ) ) )
1970  :( column()+columns() ) );
1971 
1972  for( size_t j=jbegin; j<jend; ++j )
1973  clear( matrix_(i,j) );
1974  }
1975 }
1977 //*************************************************************************************************
1978 
1979 
1980 //*************************************************************************************************
1992 template< typename MT // Type of the dense matrix
1993  , size_t... CSAs > // Compile time submatrix arguments
1995 {
1996  using blaze::clear;
1997 
1998  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
1999 
2000  const size_t jbegin( ( IsUpper_v<MT> )
2001  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
2002  ?( max( i+1UL, column() ) )
2003  :( max( i, column() ) ) )
2004  :( column() ) );
2005  const size_t jend ( ( IsLower_v<MT> )
2006  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
2007  ?( min( i, column()+columns() ) )
2008  :( min( i+1UL, column()+columns() ) ) )
2009  :( column()+columns() ) );
2010 
2011  for( size_t j=jbegin; j<jend; ++j )
2012  clear( matrix_(row()+i,j) );
2013 }
2015 //*************************************************************************************************
2016 
2017 
2018 //*************************************************************************************************
2028 template< typename MT // Type of the dense matrix
2029  , size_t... CSAs > // Compile time submatrix arguments
2030 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::hasOverlap() const noexcept
2031 {
2032  BLAZE_INTERNAL_ASSERT( IsSymmetric_v<MT> || IsHermitian_v<MT>, "Invalid matrix detected" );
2033 
2034  if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
2035  return false;
2036  else return true;
2037 }
2039 //*************************************************************************************************
2040 
2041 
2042 
2043 
2044 //=================================================================================================
2045 //
2046 // NUMERIC FUNCTIONS
2047 //
2048 //=================================================================================================
2049 
2050 //*************************************************************************************************
2068 template< typename MT // Type of the dense matrix
2069  , size_t... CSAs > // Compile time submatrix arguments
2070 inline Submatrix<MT,unaligned,false,true,CSAs...>&
2072 {
2073  if( rows() != columns() ) {
2074  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
2075  }
2076 
2077  if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
2078  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
2079  }
2080 
2081  decltype(auto) left( derestrict( *this ) );
2082  const ResultType tmp( trans( *this ) );
2083 
2084  smpAssign( left, tmp );
2085 
2086  return *this;
2087 }
2089 //*************************************************************************************************
2090 
2091 
2092 //*************************************************************************************************
2110 template< typename MT // Type of the dense matrix
2111  , size_t... CSAs > // Compile time submatrix arguments
2112 inline Submatrix<MT,unaligned,false,true,CSAs...>&
2113  Submatrix<MT,unaligned,false,true,CSAs...>::ctranspose()
2114 {
2115  if( rows() != columns() ) {
2116  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
2117  }
2118 
2119  if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
2120  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
2121  }
2122 
2123  decltype(auto) left( derestrict( *this ) );
2124  const ResultType tmp( ctrans( *this ) );
2125 
2126  smpAssign( left, tmp );
2127 
2128  return *this;
2129 }
2131 //*************************************************************************************************
2132 
2133 
2134 //*************************************************************************************************
2147 template< typename MT // Type of the dense matrix
2148  , size_t... CSAs > // Compile time submatrix arguments
2149 template< typename Other > // Data type of the scalar value
2150 inline Submatrix<MT,unaligned,false,true,CSAs...>&
2151  Submatrix<MT,unaligned,false,true,CSAs...>::scale( const Other& scalar )
2152 {
2154 
2155  const size_t iend( row() + rows() );
2156 
2157  for( size_t i=row(); i<iend; ++i )
2158  {
2159  const size_t jbegin( ( IsUpper_v<MT> )
2160  ?( ( IsStrictlyUpper_v<MT> )
2161  ?( max( i+1UL, column() ) )
2162  :( max( i, column() ) ) )
2163  :( column() ) );
2164  const size_t jend ( ( IsLower_v<MT> )
2165  ?( ( IsStrictlyLower_v<MT> )
2166  ?( min( i, column()+columns() ) )
2167  :( min( i+1UL, column()+columns() ) ) )
2168  :( column()+columns() ) );
2169 
2170  for( size_t j=jbegin; j<jend; ++j )
2171  matrix_(i,j) *= scalar;
2172  }
2173 
2174  return *this;
2175 }
2177 //*************************************************************************************************
2178 
2179 
2180 
2181 
2182 //=================================================================================================
2183 //
2184 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
2185 //
2186 //=================================================================================================
2187 
2188 //*************************************************************************************************
2199 template< typename MT // Type of the dense matrix
2200  , size_t... CSAs > // Compile time submatrix arguments
2201 template< typename Other > // Data type of the foreign expression
2202 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::canAlias( const Other* alias ) const noexcept
2203 {
2204  return matrix_.isAliased( alias );
2205 }
2207 //*************************************************************************************************
2208 
2209 
2210 //*************************************************************************************************
2221 template< typename MT // Type of the dense matrix
2222  , size_t... CSAs > // Compile time submatrix arguments
2223 template< typename MT2 // Data type of the foreign dense submatrix
2224  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
2225  , bool SO2 // Storage order of the foreign dense submatrix
2226  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
2227 inline bool
2228  Submatrix<MT,unaligned,false,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
2229 {
2230  return ( matrix_.isAliased( &alias->matrix_ ) &&
2231  ( row() + rows() > alias->row() ) &&
2232  ( row() < alias->row() + alias->rows() ) &&
2233  ( column() + columns() > alias->column() ) &&
2234  ( column() < alias->column() + alias->columns() ) );
2235 }
2237 //*************************************************************************************************
2238 
2239 
2240 //*************************************************************************************************
2251 template< typename MT // Type of the dense matrix
2252  , size_t... CSAs > // Compile time submatrix arguments
2253 template< typename Other > // Data type of the foreign expression
2254 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::isAliased( const Other* alias ) const noexcept
2255 {
2256  return matrix_.isAliased( alias );
2257 }
2259 //*************************************************************************************************
2260 
2261 
2262 //*************************************************************************************************
2273 template< typename MT // Type of the dense matrix
2274  , size_t... CSAs > // Compile time submatrix arguments
2275 template< typename MT2 // Data type of the foreign dense submatrix
2276  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
2277  , bool SO2 // Storage order of the foreign dense submatrix
2278  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
2279 inline bool
2280  Submatrix<MT,unaligned,false,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
2281 {
2282  return ( matrix_.isAliased( &alias->matrix_ ) &&
2283  ( row() + rows() > alias->row() ) &&
2284  ( row() < alias->row() + alias->rows() ) &&
2285  ( column() + columns() > alias->column() ) &&
2286  ( column() < alias->column() + alias->columns() ) );
2287 }
2289 //*************************************************************************************************
2290 
2291 
2292 //*************************************************************************************************
2302 template< typename MT // Type of the dense matrix
2303  , size_t... CSAs > // Compile time submatrix arguments
2304 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::isAligned() const noexcept
2305 {
2306  return isAligned_;
2307 }
2309 //*************************************************************************************************
2310 
2311 
2312 //*************************************************************************************************
2323 template< typename MT // Type of the dense matrix
2324  , size_t... CSAs > // Compile time submatrix arguments
2325 inline bool Submatrix<MT,unaligned,false,true,CSAs...>::canSMPAssign() const noexcept
2326 {
2327  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
2328 }
2330 //*************************************************************************************************
2331 
2332 
2333 //*************************************************************************************************
2349 template< typename MT // Type of the dense matrix
2350  , size_t... CSAs > // Compile time submatrix arguments
2351 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,false,true,CSAs...>::SIMDType
2352  Submatrix<MT,unaligned,false,true,CSAs...>::load( size_t i, size_t j ) const noexcept
2353 {
2354  if( isAligned_ )
2355  return loada( i, j );
2356  else
2357  return loadu( i, j );
2358 }
2360 //*************************************************************************************************
2361 
2362 
2363 //*************************************************************************************************
2379 template< typename MT // Type of the dense matrix
2380  , size_t... CSAs > // Compile time submatrix arguments
2381 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,false,true,CSAs...>::SIMDType
2382  Submatrix<MT,unaligned,false,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
2383 {
2385 
2386  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2387  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2388  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2389  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2390 
2391  return matrix_.loada( row()+i, column()+j );
2392 }
2394 //*************************************************************************************************
2395 
2396 
2397 //*************************************************************************************************
2413 template< typename MT // Type of the dense matrix
2414  , size_t... CSAs > // Compile time submatrix arguments
2415 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,false,true,CSAs...>::SIMDType
2416  Submatrix<MT,unaligned,false,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
2417 {
2419 
2420  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2421  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2422  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2423  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2424 
2425  return matrix_.loadu( row()+i, column()+j );
2426 }
2428 //*************************************************************************************************
2429 
2430 
2431 //*************************************************************************************************
2448 template< typename MT // Type of the dense matrix
2449  , size_t... CSAs > // Compile time submatrix arguments
2451  Submatrix<MT,unaligned,false,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
2452 {
2453  if( isAligned_ )
2454  storea( i, j, value );
2455  else
2456  storeu( i, j, value );
2457 }
2459 //*************************************************************************************************
2460 
2461 
2462 //*************************************************************************************************
2479 template< typename MT // Type of the dense matrix
2480  , size_t... CSAs > // Compile time submatrix arguments
2482  Submatrix<MT,unaligned,false,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
2483 {
2485 
2486  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2487  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2488  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2489  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2490 
2491  matrix_.storea( row()+i, column()+j, value );
2492 }
2494 //*************************************************************************************************
2495 
2496 
2497 //*************************************************************************************************
2514 template< typename MT // Type of the dense matrix
2515  , size_t... CSAs > // Compile time submatrix arguments
2517  Submatrix<MT,unaligned,false,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
2518 {
2520 
2521  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2522  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2523  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2524  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2525 
2526  matrix_.storeu( row()+i, column()+j, value );
2527 }
2529 //*************************************************************************************************
2530 
2531 
2532 //*************************************************************************************************
2549 template< typename MT // Type of the dense matrix
2550  , size_t... CSAs > // Compile time submatrix arguments
2552  Submatrix<MT,unaligned,false,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
2553 {
2555 
2556  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
2557  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
2558  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
2559  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
2560 
2561  if( isAligned_ )
2562  matrix_.stream( row()+i, column()+j, value );
2563  else
2564  matrix_.storeu( row()+i, column()+j, value );
2565 }
2567 //*************************************************************************************************
2568 
2569 
2570 //*************************************************************************************************
2582 template< typename MT // Type of the dense matrix
2583  , size_t... CSAs > // Compile time submatrix arguments
2584 template< typename MT2 > // Type of the right-hand side dense matrix
2585 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
2586  -> DisableIf_t< VectorizedAssign_v<MT2> >
2587 {
2588  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2589  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2590 
2591  const size_t jpos( columns() & size_t(-2) );
2592  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
2593 
2594  for( size_t i=0UL; i<rows(); ++i ) {
2595  for( size_t j=0UL; j<jpos; j+=2UL ) {
2596  matrix_(row()+i,column()+j ) = (~rhs)(i,j );
2597  matrix_(row()+i,column()+j+1UL) = (~rhs)(i,j+1UL);
2598  }
2599  if( jpos < columns() ) {
2600  matrix_(row()+i,column()+jpos) = (~rhs)(i,jpos);
2601  }
2602  }
2603 }
2605 //*************************************************************************************************
2606 
2607 
2608 //*************************************************************************************************
2620 template< typename MT // Type of the dense matrix
2621  , size_t... CSAs > // Compile time submatrix arguments
2622 template< typename MT2 > // Type of the right-hand side dense matrix
2623 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
2624  -> EnableIf_t< VectorizedAssign_v<MT2> >
2625 {
2627 
2628  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2629  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2630 
2631  const size_t jpos( columns() & size_t(-SIMDSIZE) );
2632  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
2633 
2634  if( useStreaming && isAligned_ &&
2635  rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
2636  !(~rhs).isAliased( &matrix_ ) )
2637  {
2638  for( size_t i=0UL; i<rows(); ++i )
2639  {
2640  size_t j( 0UL );
2641  Iterator left( begin(i) );
2642  ConstIterator_t<MT2> right( (~rhs).begin(i) );
2643 
2644  for( ; j<jpos; j+=SIMDSIZE ) {
2645  left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2646  }
2647  for( ; j<columns(); ++j ) {
2648  *left = *right;
2649  }
2650  }
2651  }
2652  else
2653  {
2654  for( size_t i=0UL; i<rows(); ++i )
2655  {
2656  size_t j( 0UL );
2657  Iterator left( begin(i) );
2658  ConstIterator_t<MT2> right( (~rhs).begin(i) );
2659 
2660  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
2661  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2662  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2663  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2664  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2665  }
2666  for( ; j<jpos; j+=SIMDSIZE ) {
2667  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2668  }
2669  for( ; j<columns(); ++j ) {
2670  *left = *right; ++left; ++right;
2671  }
2672  }
2673  }
2674 }
2676 //*************************************************************************************************
2677 
2678 
2679 //*************************************************************************************************
2691 template< typename MT // Type of the dense matrix
2692  , size_t... CSAs > // Compile time submatrix arguments
2693 template< typename MT2 > // Type of the right-hand side dense matrix
2694 inline void Submatrix<MT,unaligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
2695 {
2697 
2698  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2699  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2700 
2701  constexpr size_t block( BLOCK_SIZE );
2702 
2703  for( size_t ii=0UL; ii<rows(); ii+=block ) {
2704  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
2705  for( size_t jj=0UL; jj<columns(); jj+=block ) {
2706  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
2707  for( size_t i=ii; i<iend; ++i ) {
2708  for( size_t j=jj; j<jend; ++j ) {
2709  matrix_(row()+i,column()+j) = (~rhs)(i,j);
2710  }
2711  }
2712  }
2713  }
2714 }
2716 //*************************************************************************************************
2717 
2718 
2719 //*************************************************************************************************
2731 template< typename MT // Type of the dense matrix
2732  , size_t... CSAs > // Compile time submatrix arguments
2733 template< typename MT2 > // Type of the right-hand side sparse matrix
2734 inline void Submatrix<MT,unaligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
2735 {
2736  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2737  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2738 
2739  for( size_t i=0UL; i<rows(); ++i )
2740  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
2741  matrix_(row()+i,column()+element->index()) = element->value();
2742 }
2744 //*************************************************************************************************
2745 
2746 
2747 //*************************************************************************************************
2759 template< typename MT // Type of the dense matrix
2760  , size_t... CSAs > // Compile time submatrix arguments
2761 template< typename MT2 > // Type of the right-hand side sparse matrix
2762 inline void Submatrix<MT,unaligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
2763 {
2765 
2766  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2767  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2768 
2769  for( size_t j=0UL; j<columns(); ++j )
2770  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
2771  matrix_(row()+element->index(),column()+j) = element->value();
2772 }
2774 //*************************************************************************************************
2775 
2776 
2777 //*************************************************************************************************
2789 template< typename MT // Type of the dense matrix
2790  , size_t... CSAs > // Compile time submatrix arguments
2791 template< typename MT2 > // Type of the right-hand side dense matrix
2792 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
2793  -> DisableIf_t< VectorizedAddAssign_v<MT2> >
2794 {
2795  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2796  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2797 
2798  const size_t jpos( columns() & size_t(-2) );
2799  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
2800 
2801  for( size_t i=0UL; i<rows(); ++i )
2802  {
2803  if( IsDiagonal_v<MT2> ) {
2804  matrix_(row()+i,column()+i) += (~rhs)(i,i);
2805  }
2806  else {
2807  for( size_t j=0UL; j<jpos; j+=2UL ) {
2808  matrix_(row()+i,column()+j ) += (~rhs)(i,j );
2809  matrix_(row()+i,column()+j+1UL) += (~rhs)(i,j+1UL);
2810  }
2811  if( jpos < columns() ) {
2812  matrix_(row()+i,column()+jpos) += (~rhs)(i,jpos);
2813  }
2814  }
2815  }
2816 }
2818 //*************************************************************************************************
2819 
2820 
2821 //*************************************************************************************************
2833 template< typename MT // Type of the dense matrix
2834  , size_t... CSAs > // Compile time submatrix arguments
2835 template< typename MT2 > // Type of the right-hand side dense matrix
2836 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
2837  -> EnableIf_t< VectorizedAddAssign_v<MT2> >
2838 {
2840 
2841  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2842  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2843 
2844  for( size_t i=0UL; i<rows(); ++i )
2845  {
2846  const size_t jbegin( ( IsUpper_v<MT2> )
2847  ?( ( IsStrictlyUpper_v<MT2> ? i+1UL : i ) & size_t(-SIMDSIZE) )
2848  :( 0UL ) );
2849  const size_t jend ( ( IsLower_v<MT2> )
2850  ?( IsStrictlyLower_v<MT2> ? i : i+1UL )
2851  :( columns() ) );
2852  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
2853 
2854  const size_t jpos( jend & size_t(-SIMDSIZE) );
2855  BLAZE_INTERNAL_ASSERT( ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
2856 
2857  size_t j( jbegin );
2858  Iterator left( begin(i) + jbegin );
2859  ConstIterator_t<MT2> right( (~rhs).begin(i) + jbegin );
2860 
2861  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
2862  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2863  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2864  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2865  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2866  }
2867  for( ; j<jpos; j+=SIMDSIZE ) {
2868  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
2869  }
2870  for( ; j<jend; ++j ) {
2871  *left += *right; ++left; ++right;
2872  }
2873  }
2874 }
2876 //*************************************************************************************************
2877 
2878 
2879 //*************************************************************************************************
2891 template< typename MT // Type of the dense matrix
2892  , size_t... CSAs > // Compile time submatrix arguments
2893 template< typename MT2 > // Type of the right-hand side dense matrix
2894 inline void Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
2895 {
2897 
2898  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2899  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2900 
2901  constexpr size_t block( BLOCK_SIZE );
2902 
2903  for( size_t ii=0UL; ii<rows(); ii+=block ) {
2904  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
2905  for( size_t jj=0UL; jj<columns(); jj+=block ) {
2906  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
2907  for( size_t i=ii; i<iend; ++i ) {
2908  for( size_t j=jj; j<jend; ++j ) {
2909  matrix_(row()+i,column()+j) += (~rhs)(i,j);
2910  }
2911  }
2912  }
2913  }
2914 }
2916 //*************************************************************************************************
2917 
2918 
2919 //*************************************************************************************************
2931 template< typename MT // Type of the dense matrix
2932  , size_t... CSAs > // Compile time submatrix arguments
2933 template< typename MT2 > // Type of the right-hand side sparse matrix
2934 inline void Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
2935 {
2936  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2937  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2938 
2939  for( size_t i=0UL; i<rows(); ++i )
2940  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
2941  matrix_(row()+i,column()+element->index()) += element->value();
2942 }
2944 //*************************************************************************************************
2945 
2946 
2947 //*************************************************************************************************
2959 template< typename MT // Type of the dense matrix
2960  , size_t... CSAs > // Compile time submatrix arguments
2961 template< typename MT2 > // Type of the right-hand side sparse matrix
2962 inline void Submatrix<MT,unaligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
2963 {
2965 
2966  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2967  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2968 
2969  for( size_t j=0UL; j<columns(); ++j )
2970  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
2971  matrix_(row()+element->index(),column()+j) += element->value();
2972 }
2974 //*************************************************************************************************
2975 
2976 
2977 //*************************************************************************************************
2989 template< typename MT // Type of the dense matrix
2990  , size_t... CSAs > // Compile time submatrix arguments
2991 template< typename MT2 > // Type of the right-hand side dense matrix
2992 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
2993  -> DisableIf_t< VectorizedSubAssign_v<MT2> >
2994 {
2995  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
2996  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
2997 
2998  const size_t jpos( columns() & size_t(-2) );
2999  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
3000 
3001  for( size_t i=0UL; i<rows(); ++i )
3002  {
3003  if( IsDiagonal_v<MT2> ) {
3004  matrix_(row()+i,column()+i) -= (~rhs)(i,i);
3005  }
3006  else {
3007  for( size_t j=0UL; j<jpos; j+=2UL ) {
3008  matrix_(row()+i,column()+j ) -= (~rhs)(i,j );
3009  matrix_(row()+i,column()+j+1UL) -= (~rhs)(i,j+1UL);
3010  }
3011  if( jpos < columns() ) {
3012  matrix_(row()+i,column()+jpos) -= (~rhs)(i,jpos);
3013  }
3014  }
3015  }
3016 }
3018 //*************************************************************************************************
3019 
3020 
3021 //*************************************************************************************************
3033 template< typename MT // Type of the dense matrix
3034  , size_t... CSAs > // Compile time submatrix arguments
3035 template< typename MT2 > // Type of the right-hand side dense matrix
3036 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
3037  -> EnableIf_t< VectorizedSubAssign_v<MT2> >
3038 {
3040 
3041  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3042  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3043 
3044  for( size_t i=0UL; i<rows(); ++i )
3045  {
3046  const size_t jbegin( ( IsUpper_v<MT2> )
3047  ?( ( IsStrictlyUpper_v<MT2> ? i+1UL : i ) & size_t(-SIMDSIZE) )
3048  :( 0UL ) );
3049  const size_t jend ( ( IsLower_v<MT2> )
3050  ?( IsStrictlyLower_v<MT2> ? i : i+1UL )
3051  :( columns() ) );
3052  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
3053 
3054  const size_t jpos( jend & size_t(-SIMDSIZE) );
3055  BLAZE_INTERNAL_ASSERT( ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
3056 
3057  size_t j( jbegin );
3058  Iterator left( begin(i) + jbegin );
3059  ConstIterator_t<MT2> right( (~rhs).begin(i) + jbegin );
3060 
3061  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
3062  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3063  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3064  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3065  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3066  }
3067  for( ; j<jpos; j+=SIMDSIZE ) {
3068  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3069  }
3070  for( ; j<jend; ++j ) {
3071  *left -= *right; ++left; ++right;
3072  }
3073  }
3074 }
3076 //*************************************************************************************************
3077 
3078 
3079 //*************************************************************************************************
3091 template< typename MT // Type of the dense matrix
3092  , size_t... CSAs > // Compile time submatrix arguments
3093 template< typename MT2 > // Type of the right-hand side dense matrix
3094 inline void Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
3095 {
3097 
3098  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3099  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3100 
3101  constexpr size_t block( BLOCK_SIZE );
3102 
3103  for( size_t ii=0UL; ii<rows(); ii+=block ) {
3104  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
3105  for( size_t jj=0UL; jj<columns(); jj+=block ) {
3106  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
3107  for( size_t i=ii; i<iend; ++i ) {
3108  for( size_t j=jj; j<jend; ++j ) {
3109  matrix_(row()+i,column()+j) -= (~rhs)(i,j);
3110  }
3111  }
3112  }
3113  }
3114 }
3116 //*************************************************************************************************
3117 
3118 
3119 //*************************************************************************************************
3131 template< typename MT // Type of the dense matrix
3132  , size_t... CSAs > // Compile time submatrix arguments
3133 template< typename MT2 > // Type of the right-hand side sparse matrix
3134 inline void Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
3135 {
3136  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3137  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3138 
3139  for( size_t i=0UL; i<rows(); ++i )
3140  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
3141  matrix_(row()+i,column()+element->index()) -= element->value();
3142 }
3144 //*************************************************************************************************
3145 
3146 
3147 //*************************************************************************************************
3159 template< typename MT // Type of the dense matrix
3160  , size_t... CSAs > // Compile time submatrix arguments
3161 template< typename MT2 > // Type of the right-hand side sparse matrix
3162 inline void Submatrix<MT,unaligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
3163 {
3165 
3166  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3167  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3168 
3169  for( size_t j=0UL; j<columns(); ++j )
3170  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
3171  matrix_(row()+element->index(),column()+j) -= element->value();
3172 }
3174 //*************************************************************************************************
3175 
3176 
3177 //*************************************************************************************************
3189 template< typename MT // Type of the dense matrix
3190  , size_t... CSAs > // Compile time submatrix arguments
3191 template< typename MT2 > // Type of the right-hand side dense matrix
3192 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
3193  -> DisableIf_t< VectorizedSchurAssign_v<MT2> >
3194 {
3195  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3196  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3197 
3198  const size_t jpos( columns() & size_t(-2) );
3199  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
3200 
3201  for( size_t i=0UL; i<rows(); ++i ) {
3202  for( size_t j=0UL; j<jpos; j+=2UL ) {
3203  matrix_(row()+i,column()+j ) *= (~rhs)(i,j );
3204  matrix_(row()+i,column()+j+1UL) *= (~rhs)(i,j+1UL);
3205  }
3206  if( jpos < columns() ) {
3207  matrix_(row()+i,column()+jpos) *= (~rhs)(i,jpos);
3208  }
3209  }
3210 }
3212 //*************************************************************************************************
3213 
3214 
3215 //*************************************************************************************************
3227 template< typename MT // Type of the dense matrix
3228  , size_t... CSAs > // Compile time submatrix arguments
3229 template< typename MT2 > // Type of the right-hand side dense matrix
3230 inline auto Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
3231  -> EnableIf_t< VectorizedSchurAssign_v<MT2> >
3232 {
3234 
3235  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3236  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3237 
3238  for( size_t i=0UL; i<rows(); ++i )
3239  {
3240  const size_t jpos( columns() & size_t(-SIMDSIZE) );
3241  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
3242 
3243  size_t j( 0UL );
3244  Iterator left( begin(i) );
3245  ConstIterator_t<MT2> right( (~rhs).begin(i) );
3246 
3247  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
3248  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3249  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3250  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3251  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3252  }
3253  for( ; j<jpos; j+=SIMDSIZE ) {
3254  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
3255  }
3256  for( ; j<columns(); ++j ) {
3257  *left *= *right; ++left; ++right;
3258  }
3259  }
3260 }
3262 //*************************************************************************************************
3263 
3264 
3265 //*************************************************************************************************
3277 template< typename MT // Type of the dense matrix
3278  , size_t... CSAs > // Compile time submatrix arguments
3279 template< typename MT2 > // Type of the right-hand side dense matrix
3280 inline void Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
3281 {
3283 
3284  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3285  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3286 
3287  constexpr size_t block( BLOCK_SIZE );
3288 
3289  for( size_t ii=0UL; ii<rows(); ii+=block ) {
3290  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
3291  for( size_t jj=0UL; jj<columns(); jj+=block ) {
3292  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
3293  for( size_t i=ii; i<iend; ++i ) {
3294  for( size_t j=jj; j<jend; ++j ) {
3295  matrix_(row()+i,column()+j) *= (~rhs)(i,j);
3296  }
3297  }
3298  }
3299  }
3300 }
3302 //*************************************************************************************************
3303 
3304 
3305 //*************************************************************************************************
3317 template< typename MT // Type of the dense matrix
3318  , size_t... CSAs > // Compile time submatrix arguments
3319 template< typename MT2 > // Type of the right-hand side sparse matrix
3320 inline void Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
3321 {
3322  using blaze::reset;
3323 
3324  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3325  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3326 
3327  for( size_t i=0UL; i<rows(); ++i )
3328  {
3329  size_t j( 0UL );
3330 
3331  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element ) {
3332  for( ; j<element->index(); ++j )
3333  reset( matrix_(row()+i,column()+j) );
3334  matrix_(row()+i,column()+j) *= element->value();
3335  ++j;
3336  }
3337 
3338  for( ; j<columns(); ++j ) {
3339  reset( matrix_(row()+i,column()+j) );
3340  }
3341  }
3342 }
3344 //*************************************************************************************************
3345 
3346 
3347 //*************************************************************************************************
3359 template< typename MT // Type of the dense matrix
3360  , size_t... CSAs > // Compile time submatrix arguments
3361 template< typename MT2 > // Type of the right-hand side sparse matrix
3362 inline void Submatrix<MT,unaligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
3363 {
3364  using blaze::reset;
3365 
3367 
3368  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
3369  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
3370 
3371  for( size_t j=0UL; j<columns(); ++j )
3372  {
3373  size_t i( 0UL );
3374 
3375  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element ) {
3376  for( ; i<element->index(); ++i )
3377  reset( matrix_(row()+i,column()+j) );
3378  matrix_(row()+i,column()+j) *= element->value();
3379  ++i;
3380  }
3381 
3382  for( ; i<rows(); ++i ) {
3383  reset( matrix_(row()+i,column()+j) );
3384  }
3385  }
3386 }
3388 //*************************************************************************************************
3389 
3390 
3391 
3392 
3393 
3394 
3395 
3396 
3397 //=================================================================================================
3398 //
3399 // CLASS TEMPLATE SPECIALIZATION FOR UNALIGNED COLUMN-MAJOR DENSE SUBMATRICES
3400 //
3401 //=================================================================================================
3402 
3403 //*************************************************************************************************
3411 template< typename MT // Type of the dense matrix
3412  , size_t... CSAs > // Compile time submatrix arguments
3413 class Submatrix<MT,unaligned,true,true,CSAs...>
3414  : public View< DenseMatrix< Submatrix<MT,unaligned,true,true,CSAs...>, true > >
3415  , private SubmatrixData<CSAs...>
3416 {
3417  private:
3418  //**Type definitions****************************************************************************
3419  using DataType = SubmatrixData<CSAs...>;
3420  using Operand = If_t< IsExpression_v<MT>, MT, MT& >;
3421  //**********************************************************************************************
3422 
3423  //**********************************************************************************************
3425  template< typename MT1, typename MT2 >
3426  static constexpr bool EnforceEvaluation_v =
3427  ( IsRestricted_v<MT1> && RequiresEvaluation_v<MT2> );
3428  //**********************************************************************************************
3429 
3430  public:
3431  //**Type definitions****************************************************************************
3433  using This = Submatrix<MT,unaligned,true,true,CSAs...>;
3434 
3435  using BaseType = DenseMatrix<This,true>;
3436  using ViewedType = MT;
3437  using ResultType = SubmatrixTrait_t<MT,CSAs...>;
3438  using OppositeType = OppositeType_t<ResultType>;
3439  using TransposeType = TransposeType_t<ResultType>;
3440  using ElementType = ElementType_t<MT>;
3441  using SIMDType = SIMDTrait_t<ElementType>;
3442  using ReturnType = ReturnType_t<MT>;
3443  using CompositeType = const Submatrix&;
3444 
3446  using ConstReference = ConstReference_t<MT>;
3447 
3449  using Reference = If_t< IsConst_v<MT>, ConstReference, Reference_t<MT> >;
3450 
3452  using ConstPointer = ConstPointer_t<MT>;
3453 
3455  using Pointer = If_t< IsConst_v<MT> || !HasMutableDataAccess_v<MT>, ConstPointer, Pointer_t<MT> >;
3456  //**********************************************************************************************
3457 
3458  //**SubmatrixIterator class definition**********************************************************
3461  template< typename IteratorType > // Type of the dense matrix iterator
3462  class SubmatrixIterator
3463  {
3464  public:
3465  //**Type definitions*************************************************************************
3467  using IteratorCategory = typename std::iterator_traits<IteratorType>::iterator_category;
3468 
3470  using ValueType = typename std::iterator_traits<IteratorType>::value_type;
3471 
3473  using PointerType = typename std::iterator_traits<IteratorType>::pointer;
3474 
3476  using ReferenceType = typename std::iterator_traits<IteratorType>::reference;
3477 
3479  using DifferenceType = typename std::iterator_traits<IteratorType>::difference_type;
3480 
3481  // STL iterator requirements
3482  using iterator_category = IteratorCategory;
3483  using value_type = ValueType;
3484  using pointer = PointerType;
3485  using reference = ReferenceType;
3486  using difference_type = DifferenceType;
3487  //*******************************************************************************************
3488 
3489  //**Constructor******************************************************************************
3492  inline SubmatrixIterator()
3493  : iterator_ ( ) // Iterator to the current submatrix element
3494  , isAligned_( false ) // Memory alignment flag
3495  {}
3496  //*******************************************************************************************
3497 
3498  //**Constructor******************************************************************************
3506  inline SubmatrixIterator( IteratorType iterator, bool isMemoryAligned )
3507  : iterator_ ( iterator ) // Iterator to the current submatrix element
3508  , isAligned_( isMemoryAligned ) // Memory alignment flag
3509  {}
3510  //*******************************************************************************************
3511 
3512  //**Constructor******************************************************************************
3517  template< typename IteratorType2 >
3518  inline SubmatrixIterator( const SubmatrixIterator<IteratorType2>& it )
3519  : iterator_ ( it.base() ) // Iterator to the current submatrix element
3520  , isAligned_( it.isAligned() ) // Memory alignment flag
3521  {}
3522  //*******************************************************************************************
3523 
3524  //**Addition assignment operator*************************************************************
3530  inline SubmatrixIterator& operator+=( size_t inc ) {
3531  iterator_ += inc;
3532  return *this;
3533  }
3534  //*******************************************************************************************
3535 
3536  //**Subtraction assignment operator**********************************************************
3542  inline SubmatrixIterator& operator-=( size_t dec ) {
3543  iterator_ -= dec;
3544  return *this;
3545  }
3546  //*******************************************************************************************
3547 
3548  //**Prefix increment operator****************************************************************
3553  inline SubmatrixIterator& operator++() {
3554  ++iterator_;
3555  return *this;
3556  }
3557  //*******************************************************************************************
3558 
3559  //**Postfix increment operator***************************************************************
3564  inline const SubmatrixIterator operator++( int ) {
3565  return SubmatrixIterator( iterator_++, isAligned_ );
3566  }
3567  //*******************************************************************************************
3568 
3569  //**Prefix decrement operator****************************************************************
3574  inline SubmatrixIterator& operator--() {
3575  --iterator_;
3576  return *this;
3577  }
3578  //*******************************************************************************************
3579 
3580  //**Postfix decrement operator***************************************************************
3585  inline const SubmatrixIterator operator--( int ) {
3586  return SubmatrixIterator( iterator_--, isAligned_ );
3587  }
3588  //*******************************************************************************************
3589 
3590  //**Element access operator******************************************************************
3595  inline ReferenceType operator*() const {
3596  return *iterator_;
3597  }
3598  //*******************************************************************************************
3599 
3600  //**Element access operator******************************************************************
3605  inline IteratorType operator->() const {
3606  return iterator_;
3607  }
3608  //*******************************************************************************************
3609 
3610  //**Load function****************************************************************************
3620  inline SIMDType load() const noexcept {
3621  if( isAligned_ )
3622  return loada();
3623  else
3624  return loadu();
3625  }
3626  //*******************************************************************************************
3627 
3628  //**Loada function***************************************************************************
3638  inline SIMDType loada() const noexcept {
3639  return iterator_.loada();
3640  }
3641  //*******************************************************************************************
3642 
3643  //**Loadu function***************************************************************************
3653  inline SIMDType loadu() const noexcept {
3654  return iterator_.loadu();
3655  }
3656  //*******************************************************************************************
3657 
3658  //**Store function***************************************************************************
3669  inline void store( const SIMDType& value ) const {
3670  if( isAligned_ ) {
3671  storea( value );
3672  }
3673  else {
3674  storeu( value );
3675  }
3676  }
3677  //*******************************************************************************************
3678 
3679  //**Storea function**************************************************************************
3690  inline void storea( const SIMDType& value ) const {
3691  iterator_.storea( value );
3692  }
3693  //*******************************************************************************************
3694 
3695  //**Storeu function**************************************************************************
3706  inline void storeu( const SIMDType& value ) const {
3707  iterator_.storeu( value );
3708  }
3709  //*******************************************************************************************
3710 
3711  //**Stream function**************************************************************************
3722  inline void stream( const SIMDType& value ) const {
3723  iterator_.stream( value );
3724  }
3725  //*******************************************************************************************
3726 
3727  //**Equality operator************************************************************************
3733  inline bool operator==( const SubmatrixIterator& rhs ) const {
3734  return iterator_ == rhs.iterator_;
3735  }
3736  //*******************************************************************************************
3737 
3738  //**Inequality operator**********************************************************************
3744  inline bool operator!=( const SubmatrixIterator& rhs ) const {
3745  return iterator_ != rhs.iterator_;
3746  }
3747  //*******************************************************************************************
3748 
3749  //**Less-than operator***********************************************************************
3755  inline bool operator<( const SubmatrixIterator& rhs ) const {
3756  return iterator_ < rhs.iterator_;
3757  }
3758  //*******************************************************************************************
3759 
3760  //**Greater-than operator********************************************************************
3766  inline bool operator>( const SubmatrixIterator& rhs ) const {
3767  return iterator_ > rhs.iterator_;
3768  }
3769  //*******************************************************************************************
3770 
3771  //**Less-or-equal-than operator**************************************************************
3777  inline bool operator<=( const SubmatrixIterator& rhs ) const {
3778  return iterator_ <= rhs.iterator_;
3779  }
3780  //*******************************************************************************************
3781 
3782  //**Greater-or-equal-than operator***********************************************************
3788  inline bool operator>=( const SubmatrixIterator& rhs ) const {
3789  return iterator_ >= rhs.iterator_;
3790  }
3791  //*******************************************************************************************
3792 
3793  //**Subtraction operator*********************************************************************
3799  inline DifferenceType operator-( const SubmatrixIterator& rhs ) const {
3800  return iterator_ - rhs.iterator_;
3801  }
3802  //*******************************************************************************************
3803 
3804  //**Addition operator************************************************************************
3811  friend inline const SubmatrixIterator operator+( const SubmatrixIterator& it, size_t inc ) {
3812  return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
3813  }
3814  //*******************************************************************************************
3815 
3816  //**Addition operator************************************************************************
3823  friend inline const SubmatrixIterator operator+( size_t inc, const SubmatrixIterator& it ) {
3824  return SubmatrixIterator( it.iterator_ + inc, it.isAligned_ );
3825  }
3826  //*******************************************************************************************
3827 
3828  //**Subtraction operator*********************************************************************
3835  friend inline const SubmatrixIterator operator-( const SubmatrixIterator& it, size_t dec ) {
3836  return SubmatrixIterator( it.iterator_ - dec, it.isAligned_ );
3837  }
3838  //*******************************************************************************************
3839 
3840  //**Base function****************************************************************************
3845  inline IteratorType base() const {
3846  return iterator_;
3847  }
3848  //*******************************************************************************************
3849 
3850  //**IsAligned function***********************************************************************
3855  inline bool isAligned() const noexcept {
3856  return isAligned_;
3857  }
3858  //*******************************************************************************************
3859 
3860  private:
3861  //**Member variables*************************************************************************
3862  IteratorType iterator_;
3863  bool isAligned_;
3864  //*******************************************************************************************
3865  };
3866  //**********************************************************************************************
3867 
3868  //**Type definitions****************************************************************************
3870  using ConstIterator = SubmatrixIterator< ConstIterator_t<MT> >;
3871 
3873  using Iterator = If_t< IsConst_v<MT>, ConstIterator, SubmatrixIterator< Iterator_t<MT> > >;
3874  //**********************************************************************************************
3875 
3876  //**Compilation flags***************************************************************************
3878  static constexpr bool simdEnabled = MT::simdEnabled;
3879 
3881  static constexpr bool smpAssignable = MT::smpAssignable;
3882  //**********************************************************************************************
3883 
3884  //**Constructors********************************************************************************
3887  template< typename... RSAs >
3888  explicit inline Submatrix( MT& matrix, RSAs... args );
3889 
3890  Submatrix( const Submatrix& ) = default;
3892  //**********************************************************************************************
3893 
3894  //**Destructor**********************************************************************************
3897  ~Submatrix() = default;
3899  //**********************************************************************************************
3900 
3901  //**Data access functions***********************************************************************
3904  inline Reference operator()( size_t i, size_t j );
3905  inline ConstReference operator()( size_t i, size_t j ) const;
3906  inline Reference at( size_t i, size_t j );
3907  inline ConstReference at( size_t i, size_t j ) const;
3908  inline Pointer data () noexcept;
3909  inline ConstPointer data () const noexcept;
3910  inline Pointer data ( size_t j ) noexcept;
3911  inline ConstPointer data ( size_t j ) const noexcept;
3912  inline Iterator begin ( size_t j );
3913  inline ConstIterator begin ( size_t j ) const;
3914  inline ConstIterator cbegin( size_t j ) const;
3915  inline Iterator end ( size_t j );
3916  inline ConstIterator end ( size_t j ) const;
3917  inline ConstIterator cend ( size_t j ) const;
3919  //**********************************************************************************************
3920 
3921  //**Assignment operators************************************************************************
3924  inline Submatrix& operator=( const ElementType& rhs );
3925  inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
3926  inline Submatrix& operator=( const Submatrix& rhs );
3927 
3928  template< typename MT2, bool SO >
3929  inline Submatrix& operator=( const Matrix<MT2,SO>& rhs );
3930 
3931  template< typename MT2, bool SO >
3932  inline auto operator+=( const Matrix<MT2,SO>& rhs )
3933  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3934 
3935  template< typename MT2, bool SO >
3936  inline auto operator+=( const Matrix<MT2,SO>& rhs )
3937  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3938 
3939  template< typename MT2, bool SO >
3940  inline auto operator-=( const Matrix<MT2,SO>& rhs )
3941  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3942 
3943  template< typename MT2, bool SO >
3944  inline auto operator-=( const Matrix<MT2,SO>& rhs )
3945  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3946 
3947  template< typename MT2, bool SO >
3948  inline auto operator%=( const Matrix<MT2,SO>& rhs )
3949  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3950 
3951  template< typename MT2, bool SO >
3952  inline auto operator%=( const Matrix<MT2,SO>& rhs )
3953  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
3955  //**********************************************************************************************
3956 
3957  //**Utility functions***************************************************************************
3960  using DataType::row;
3961  using DataType::column;
3962  using DataType::rows;
3963  using DataType::columns;
3964 
3965  inline MT& operand() noexcept;
3966  inline const MT& operand() const noexcept;
3967 
3968  inline size_t spacing() const noexcept;
3969  inline size_t capacity() const noexcept;
3970  inline size_t capacity( size_t i ) const noexcept;
3971  inline size_t nonZeros() const;
3972  inline size_t nonZeros( size_t i ) const;
3973  inline void reset();
3974  inline void reset( size_t i );
3976  //**********************************************************************************************
3977 
3978  //**Numeric functions***************************************************************************
3981  inline Submatrix& transpose();
3982  inline Submatrix& ctranspose();
3983 
3984  template< typename Other > inline Submatrix& scale( const Other& scalar );
3986  //**********************************************************************************************
3987 
3988  private:
3989  //**********************************************************************************************
3991  template< typename MT2 >
3992  static constexpr bool VectorizedAssign_v =
3993  ( useOptimizedKernels &&
3994  simdEnabled && MT2::simdEnabled &&
3996  //**********************************************************************************************
3997 
3998  //**********************************************************************************************
4000  template< typename MT2 >
4001  static constexpr bool VectorizedAddAssign_v =
4002  ( useOptimizedKernels &&
4003  simdEnabled && MT2::simdEnabled &&
4006  !IsDiagonal_v<MT2> );
4007  //**********************************************************************************************
4008 
4009  //**********************************************************************************************
4011  template< typename MT2 >
4012  static constexpr bool VectorizedSubAssign_v =
4013  ( useOptimizedKernels &&
4014  simdEnabled && MT2::simdEnabled &&
4017  !IsDiagonal_v<MT2> );
4018  //**********************************************************************************************
4019 
4020  //**********************************************************************************************
4022  template< typename MT2 >
4023  static constexpr bool VectorizedSchurAssign_v =
4024  ( useOptimizedKernels &&
4025  simdEnabled && MT2::simdEnabled &&
4028  //**********************************************************************************************
4029 
4030  //**SIMD properties*****************************************************************************
4032  static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
4033  //**********************************************************************************************
4034 
4035  public:
4036  //**Expression template evaluation functions****************************************************
4039  template< typename Other >
4040  inline bool canAlias( const Other* alias ) const noexcept;
4041 
4042  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
4043  inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
4044 
4045  template< typename Other >
4046  inline bool isAliased( const Other* alias ) const noexcept;
4047 
4048  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
4049  inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
4050 
4051  inline bool isAligned () const noexcept;
4052  inline bool canSMPAssign() const noexcept;
4053 
4054  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
4055  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
4056  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
4057 
4058  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
4059  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
4060  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
4061  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
4062 
4063  template< typename MT2 >
4064  inline auto assign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT2> >;
4065 
4066  template< typename MT2 >
4067  inline auto assign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT2> >;
4068 
4069  template< typename MT2 > inline void assign( const DenseMatrix<MT2,false>& rhs );
4070  template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
4071  template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
4072 
4073  template< typename MT2 >
4074  inline auto addAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT2> >;
4075 
4076  template< typename MT2 >
4077  inline auto addAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT2> >;
4078 
4079  template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,false>& rhs );
4080  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
4081  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
4082 
4083  template< typename MT2 >
4084  inline auto subAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT2> >;
4085 
4086  template< typename MT2 >
4087  inline auto subAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT2> >;
4088 
4089  template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,false>& rhs );
4090  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
4091  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
4092 
4093  template< typename MT2 >
4094  inline auto schurAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT2> >;
4095 
4096  template< typename MT2 >
4097  inline auto schurAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT2> >;
4098 
4099  template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,false>& rhs );
4100  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
4101  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
4103  //**********************************************************************************************
4104 
4105  private:
4106  //**Utility functions***************************************************************************
4109  inline bool hasOverlap() const noexcept;
4111  //**********************************************************************************************
4112 
4113  //**Member variables****************************************************************************
4116  Operand matrix_;
4117  const bool isAligned_;
4118 
4125  //**********************************************************************************************
4126 
4127  //**Friend declarations*************************************************************************
4128  template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
4129  //**********************************************************************************************
4130 
4131  //**Compile time checks*************************************************************************
4139  //**********************************************************************************************
4140 };
4142 //*************************************************************************************************
4143 
4144 
4145 
4146 
4147 //=================================================================================================
4148 //
4149 // CONSTRUCTORS
4150 //
4151 //=================================================================================================
4152 
4153 //*************************************************************************************************
4166 template< typename MT // Type of the dense matrix
4167  , size_t... CSAs > // Compile time submatrix arguments
4168 template< typename... RSAs > // Runtime submatrix arguments
4169 inline Submatrix<MT,unaligned,true,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
4170  : DataType ( args... ) // Base class initialization
4171  , matrix_ ( matrix ) // The matrix containing the submatrix
4172  , isAligned_( simdEnabled && IsContiguous_v<MT> &&
4173  matrix.data() != nullptr && checkAlignment( data() ) &&
4174  ( columns() < 2UL || ( matrix.spacing() & size_t(-SIMDSIZE) ) == 0UL ) )
4175 {
4176  if( !Contains_v< TypeList<RSAs...>, Unchecked > ) {
4177  if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
4178  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
4179  }
4180  }
4181  else {
4182  BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
4183  BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
4184  }
4185 }
4187 //*************************************************************************************************
4188 
4189 
4190 
4191 
4192 //=================================================================================================
4193 //
4194 // DATA ACCESS FUNCTIONS
4195 //
4196 //=================================================================================================
4197 
4198 //*************************************************************************************************
4209 template< typename MT // Type of the dense matrix
4210  , size_t... CSAs > // Compile time submatrix arguments
4211 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Reference
4212  Submatrix<MT,unaligned,true,true,CSAs...>::operator()( size_t i, size_t j )
4213 {
4214  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
4215  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
4216 
4217  return matrix_(row()+i,column()+j);
4218 }
4220 //*************************************************************************************************
4221 
4222 
4223 //*************************************************************************************************
4234 template< typename MT // Type of the dense matrix
4235  , size_t... CSAs > // Compile time submatrix arguments
4236 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstReference
4237  Submatrix<MT,unaligned,true,true,CSAs...>::operator()( size_t i, size_t j ) const
4238 {
4239  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
4240  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
4241 
4242  return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
4243 }
4245 //*************************************************************************************************
4246 
4247 
4248 //*************************************************************************************************
4260 template< typename MT // Type of the dense matrix
4261  , size_t... CSAs > // Compile time submatrix arguments
4262 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Reference
4263  Submatrix<MT,unaligned,true,true,CSAs...>::at( size_t i, size_t j )
4264 {
4265  if( i >= rows() ) {
4266  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
4267  }
4268  if( j >= columns() ) {
4269  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
4270  }
4271  return (*this)(i,j);
4272 }
4274 //*************************************************************************************************
4275 
4276 
4277 //*************************************************************************************************
4289 template< typename MT // Type of the dense matrix
4290  , size_t... CSAs > // Compile time submatrix arguments
4291 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstReference
4292  Submatrix<MT,unaligned,true,true,CSAs...>::at( size_t i, size_t j ) const
4293 {
4294  if( i >= rows() ) {
4295  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
4296  }
4297  if( j >= columns() ) {
4298  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
4299  }
4300  return (*this)(i,j);
4301 }
4303 //*************************************************************************************************
4304 
4305 
4306 //*************************************************************************************************
4316 template< typename MT // Type of the dense matrix
4317  , size_t... CSAs > // Compile time submatrix arguments
4318 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Pointer
4320 {
4321  return matrix_.data() + row() + column()*spacing();
4322 }
4324 //*************************************************************************************************
4325 
4326 
4327 //*************************************************************************************************
4337 template< typename MT // Type of the dense matrix
4338  , size_t... CSAs > // Compile time submatrix arguments
4339 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstPointer
4341 {
4342  return matrix_.data() + row() + column()*spacing();
4343 }
4345 //*************************************************************************************************
4346 
4347 
4348 //*************************************************************************************************
4357 template< typename MT // Type of the dense matrix
4358  , size_t... CSAs > // Compile time submatrix arguments
4359 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Pointer
4361 {
4362  return matrix_.data() + row() + (column()+j)*spacing();
4363 }
4365 //*************************************************************************************************
4366 
4367 
4368 //*************************************************************************************************
4377 template< typename MT // Type of the dense matrix
4378  , size_t... CSAs > // Compile time submatrix arguments
4379 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstPointer
4380  Submatrix<MT,unaligned,true,true,CSAs...>::data( size_t j ) const noexcept
4381 {
4382  return matrix_.data() + row() + (column()+j)*spacing();
4383 }
4385 //*************************************************************************************************
4386 
4387 
4388 //*************************************************************************************************
4395 template< typename MT // Type of the dense matrix
4396  , size_t... CSAs > // Compile time submatrix arguments
4397 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Iterator
4399 {
4400  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4401  return Iterator( matrix_.begin( column() + j ) + row(), isAligned_ );
4402 }
4404 //*************************************************************************************************
4405 
4406 
4407 //*************************************************************************************************
4414 template< typename MT // Type of the dense matrix
4415  , size_t... CSAs > // Compile time submatrix arguments
4416 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4418 {
4419  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4420  return ConstIterator( matrix_.cbegin( column() + j ) + row(), isAligned_ );
4421 }
4423 //*************************************************************************************************
4424 
4425 
4426 //*************************************************************************************************
4433 template< typename MT // Type of the dense matrix
4434  , size_t... CSAs > // Compile time submatrix arguments
4435 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4437 {
4438  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4439  return ConstIterator( matrix_.cbegin( column() + j ) + row(), isAligned_ );
4440 }
4442 //*************************************************************************************************
4443 
4444 
4445 //*************************************************************************************************
4452 template< typename MT // Type of the dense matrix
4453  , size_t... CSAs > // Compile time submatrix arguments
4454 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::Iterator
4456 {
4457  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4458  return Iterator( matrix_.begin( column() + j ) + row() + rows(), isAligned_ );
4459 }
4461 //*************************************************************************************************
4462 
4463 
4464 //*************************************************************************************************
4471 template< typename MT // Type of the dense matrix
4472  , size_t... CSAs > // Compile time submatrix arguments
4473 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4475 {
4476  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4477  return ConstIterator( matrix_.cbegin( column() + j ) + row() + rows(), isAligned_ );
4478 }
4480 //*************************************************************************************************
4481 
4482 
4483 //*************************************************************************************************
4490 template< typename MT // Type of the dense matrix
4491  , size_t... CSAs > // Compile time submatrix arguments
4492 inline typename Submatrix<MT,unaligned,true,true,CSAs...>::ConstIterator
4494 {
4495  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
4496  return ConstIterator( matrix_.cbegin( column() + j ) + row() + rows(), isAligned_ );
4497 }
4499 //*************************************************************************************************
4500 
4501 
4502 
4503 
4504 //=================================================================================================
4505 //
4506 // ASSIGNMENT OPERATORS
4507 //
4508 //=================================================================================================
4509 
4510 //*************************************************************************************************
4521 template< typename MT // Type of the dense matrix
4522  , size_t... CSAs > // Compile time submatrix arguments
4523 inline Submatrix<MT,unaligned,true,true,CSAs...>&
4524  Submatrix<MT,unaligned,true,true,CSAs...>::operator=( const ElementType& rhs )
4525 {
4526  const size_t jend( column() + columns() );
4527  decltype(auto) left( derestrict( matrix_ ) );
4528 
4529  for( size_t j=column(); j<jend; ++j )
4530  {
4531  const size_t ibegin( ( IsLower_v<MT> )
4532  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
4533  ?( max( j+1UL, row() ) )
4534  :( max( j, row() ) ) )
4535  :( row() ) );
4536  const size_t iend ( ( IsUpper_v<MT> )
4537  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
4538  ?( min( j, row()+rows() ) )
4539  :( min( j+1UL, row()+rows() ) ) )
4540  :( row()+rows() ) );
4541 
4542  for( size_t i=ibegin; i<iend; ++i ) {
4543  if( !IsRestricted_v<MT> || IsTriangular_v<MT> || trySet( matrix_, i, j, rhs ) )
4544  left(i,j) = rhs;
4545  }
4546  }
4547 
4548  return *this;
4549 }
4551 //*************************************************************************************************
4552 
4553 
4554 //*************************************************************************************************
4570 template< typename MT // Type of the dense matrix
4571  , size_t... CSAs > // Compile time submatrix arguments
4572 inline Submatrix<MT,unaligned,true,true,CSAs...>&
4573  Submatrix<MT,unaligned,true,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
4574 {
4575  using blaze::reset;
4576 
4577  if( list.size() != rows() ) {
4578  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
4579  }
4580 
4581  if( IsRestricted_v<MT> ) {
4582  const InitializerMatrix<ElementType> tmp( list, columns() );
4583  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
4584  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4585  }
4586  }
4587 
4588  decltype(auto) left( derestrict( *this ) );
4589  size_t i( 0UL );
4590 
4591  for( const auto& rowList : list ) {
4592  size_t j( 0UL );
4593  for( const auto& element : rowList ) {
4594  left(i,j) = element;
4595  ++j;
4596  }
4597  for( ; j<columns(); ++j ) {
4598  reset( left(i,j) );
4599  }
4600  ++i;
4601  }
4602 
4603  return *this;
4604 }
4606 //*************************************************************************************************
4607 
4608 
4609 //*************************************************************************************************
4624 template< typename MT // Type of the dense matrix
4625  , size_t... CSAs > // Compile time submatrix arguments
4626 inline Submatrix<MT,unaligned,true,true,CSAs...>&
4627  Submatrix<MT,unaligned,true,true,CSAs...>::operator=( const Submatrix& rhs )
4628 {
4631 
4632  if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
4633  return *this;
4634 
4635  if( rows() != rhs.rows() || columns() != rhs.columns() ) {
4636  BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
4637  }
4638 
4639  if( !tryAssign( matrix_, rhs, row(), column() ) ) {
4640  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4641  }
4642 
4643  decltype(auto) left( derestrict( *this ) );
4644 
4645  if( rhs.canAlias( &matrix_ ) ) {
4646  const ResultType tmp( rhs );
4647  smpAssign( left, tmp );
4648  }
4649  else {
4650  smpAssign( left, rhs );
4651  }
4652 
4653  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4654 
4655  return *this;
4656 }
4658 //*************************************************************************************************
4659 
4660 
4661 //*************************************************************************************************
4676 template< typename MT // Type of the dense matrix
4677  , size_t... CSAs > // Compile time submatrix arguments
4678 template< typename MT2 // Type of the right-hand side matrix
4679  , bool SO > // Storage order of the right-hand side matrix
4680 inline Submatrix<MT,unaligned,true,true,CSAs...>&
4681  Submatrix<MT,unaligned,true,true,CSAs...>::operator=( const Matrix<MT2,SO>& rhs )
4682 {
4683  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
4684 
4685  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4686  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4687  }
4688 
4689  using Right = If_t< IsRestricted_v<MT>, CompositeType_t<MT2>, const MT2& >;
4690  Right right( ~rhs );
4691 
4692  if( !tryAssign( matrix_, right, row(), column() ) ) {
4693  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4694  }
4695 
4696  decltype(auto) left( derestrict( *this ) );
4697 
4698  if( IsReference_v<Right> && right.canAlias( &matrix_ ) ) {
4699  const ResultType_t<MT2> tmp( right );
4700  if( IsSparseMatrix_v<MT2> )
4701  reset();
4702  smpAssign( left, tmp );
4703  }
4704  else {
4705  if( IsSparseMatrix_v<MT2> )
4706  reset();
4707  smpAssign( left, right );
4708  }
4709 
4710  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4711 
4712  return *this;
4713 }
4715 //*************************************************************************************************
4716 
4717 
4718 //*************************************************************************************************
4732 template< typename MT // Type of the dense matrix
4733  , size_t... CSAs > // Compile time submatrix arguments
4734 template< typename MT2 // Type of the right-hand side matrix
4735  , bool SO > // Storage order of the right-hand side matrix
4736 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
4737  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4738 {
4741  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
4742 
4743  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
4744 
4747 
4748  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4749  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4750  }
4751 
4752  if( !tryAddAssign( matrix_, ~rhs, row(), column() ) ) {
4753  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4754  }
4755 
4756  decltype(auto) left( derestrict( *this ) );
4757 
4758  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
4759  (~rhs).canAlias( &matrix_ ) ) {
4760  const AddType tmp( *this + (~rhs) );
4761  smpAssign( left, tmp );
4762  }
4763  else {
4764  smpAddAssign( left, ~rhs );
4765  }
4766 
4767  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4768 
4769  return *this;
4770 }
4772 //*************************************************************************************************
4773 
4774 
4775 //*************************************************************************************************
4789 template< typename MT // Type of the dense matrix
4790  , size_t... CSAs > // Compile time submatrix arguments
4791 template< typename MT2 // Type of the right-hand side matrix
4792  , bool SO > // Storage order of the right-hand side matrix
4793 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
4794  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4795 {
4798  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
4799 
4800  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
4801 
4804 
4805  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4806  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4807  }
4808 
4809  const AddType tmp( *this + (~rhs) );
4810 
4811  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
4812  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4813  }
4814 
4815  decltype(auto) left( derestrict( *this ) );
4816 
4817  smpAssign( left, tmp );
4818 
4819  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4820 
4821  return *this;
4822 }
4824 //*************************************************************************************************
4825 
4826 
4827 //*************************************************************************************************
4841 template< typename MT // Type of the dense matrix
4842  , size_t... CSAs > // Compile time submatrix arguments
4843 template< typename MT2 // Type of the right-hand side matrix
4844  , bool SO > // Storage order of the right-hand side matrix
4845 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
4846  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4847 {
4850  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
4851 
4852  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
4853 
4856 
4857  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4858  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4859  }
4860 
4861  if( !trySubAssign( matrix_, ~rhs, row(), column() ) ) {
4862  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4863  }
4864 
4865  decltype(auto) left( derestrict( *this ) );
4866 
4867  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
4868  (~rhs).canAlias( &matrix_ ) ) {
4869  const SubType tmp( *this - (~rhs ) );
4870  smpAssign( left, tmp );
4871  }
4872  else {
4873  smpSubAssign( left, ~rhs );
4874  }
4875 
4876  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4877 
4878  return *this;
4879 }
4881 //*************************************************************************************************
4882 
4883 
4884 //*************************************************************************************************
4898 template< typename MT // Type of the dense matrix
4899  , size_t... CSAs > // Compile time submatrix arguments
4900 template< typename MT2 // Type of the right-hand side matrix
4901  , bool SO > // Storage order of the right-hand side matrix
4902 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
4903  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4904 {
4907  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
4908 
4909  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
4910 
4913 
4914  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4915  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4916  }
4917 
4918  const SubType tmp( *this - (~rhs) );
4919 
4920  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
4921  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4922  }
4923 
4924  decltype(auto) left( derestrict( *this ) );
4925 
4926  smpAssign( left, tmp );
4927 
4928  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4929 
4930  return *this;
4931 }
4933 //*************************************************************************************************
4934 
4935 
4936 //*************************************************************************************************
4950 template< typename MT // Type of the dense matrix
4951  , size_t... CSAs > // Compile time submatrix arguments
4952 template< typename MT2 // Type of the right-hand side matrix
4953  , bool SO > // Storage order of the right-hand side matrix
4954 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
4955  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
4956 {
4959  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
4960 
4961  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
4962 
4964 
4965  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
4966  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
4967  }
4968 
4969  if( !trySchurAssign( matrix_, ~rhs, row(), column() ) ) {
4970  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
4971  }
4972 
4973  decltype(auto) left( derestrict( *this ) );
4974 
4975  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
4976  (~rhs).canAlias( &matrix_ ) ) {
4977  const SchurType tmp( *this % (~rhs) );
4978  if( IsSparseMatrix_v<SchurType> )
4979  reset();
4980  smpAssign( left, tmp );
4981  }
4982  else {
4983  smpSchurAssign( left, ~rhs );
4984  }
4985 
4986  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
4987 
4988  return *this;
4989 }
4991 //*************************************************************************************************
4992 
4993 
4994 //*************************************************************************************************
5008 template< typename MT // Type of the dense matrix
5009  , size_t... CSAs > // Compile time submatrix arguments
5010 template< typename MT2 // Type of the right-hand side matrix
5011  , bool SO > // Storage order of the right-hand side matrix
5012 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
5013  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
5014 {
5017  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
5018 
5019  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
5020 
5022 
5023  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
5024  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
5025  }
5026 
5027  const SchurType tmp( *this % (~rhs) );
5028 
5029  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
5030  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
5031  }
5032 
5033  decltype(auto) left( derestrict( *this ) );
5034 
5035  if( IsSparseMatrix_v<SchurType> ) {
5036  reset();
5037  }
5038 
5039  smpAssign( left, tmp );
5040 
5041  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
5042 
5043  return *this;
5044 }
5046 //*************************************************************************************************
5047 
5048 
5049 
5050 
5051 //=================================================================================================
5052 //
5053 // UTILITY FUNCTIONS
5054 //
5055 //=================================================================================================
5056 
5057 //*************************************************************************************************
5063 template< typename MT // Type of the dense matrix
5064  , size_t... CSAs > // Compile time submatrix arguments
5065 inline MT& Submatrix<MT,unaligned,true,true,CSAs...>::operand() noexcept
5066 {
5067  return matrix_;
5068 }
5070 //*************************************************************************************************
5071 
5072 
5073 //*************************************************************************************************
5079 template< typename MT // Type of the dense matrix
5080  , size_t... CSAs > // Compile time submatrix arguments
5081 inline const MT& Submatrix<MT,unaligned,true,true,CSAs...>::operand() const noexcept
5082 {
5083  return matrix_;
5084 }
5086 //*************************************************************************************************
5087 
5088 
5089 //*************************************************************************************************
5098 template< typename MT // Type of the dense matrix
5099  , size_t... CSAs > // Compile time submatrix arguments
5100 inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::spacing() const noexcept
5101 {
5102  return matrix_.spacing();
5103 }
5105 //*************************************************************************************************
5106 
5107 
5108 //*************************************************************************************************
5114 template< typename MT // Type of the dense matrix
5115  , size_t... CSAs > // Compile time submatrix arguments
5116 inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::capacity() const noexcept
5117 {
5118  return rows() * columns();
5119 }
5121 //*************************************************************************************************
5122 
5123 
5124 //*************************************************************************************************
5131 template< typename MT // Type of the dense matrix
5132  , size_t... CSAs > // Compile time submatrix arguments
5133 inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::capacity( size_t j ) const noexcept
5134 {
5135  UNUSED_PARAMETER( j );
5136 
5137  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
5138 
5139  return rows();
5140 }
5142 //*************************************************************************************************
5143 
5144 
5145 //*************************************************************************************************
5151 template< typename MT // Type of the dense matrix
5152  , size_t... CSAs > // Compile time submatrix arguments
5154 {
5155  const size_t iend( row() + rows() );
5156  const size_t jend( column() + columns() );
5157  size_t nonzeros( 0UL );
5158 
5159  for( size_t j=column(); j<jend; ++j )
5160  for( size_t i=row(); i<iend; ++i )
5161  if( !isDefault( matrix_(i,j) ) )
5162  ++nonzeros;
5163 
5164  return nonzeros;
5165 }
5167 //*************************************************************************************************
5168 
5169 
5170 //*************************************************************************************************
5177 template< typename MT // Type of the dense matrix
5178  , size_t... CSAs > // Compile time submatrix arguments
5179 inline size_t Submatrix<MT,unaligned,true,true,CSAs...>::nonZeros( size_t j ) const
5180 {
5181  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
5182 
5183  const size_t iend( row() + rows() );
5184  size_t nonzeros( 0UL );
5185 
5186  for( size_t i=row(); i<iend; ++i )
5187  if( !isDefault( matrix_(i,column()+j) ) )
5188  ++nonzeros;
5189 
5190  return nonzeros;
5191 }
5193 //*************************************************************************************************
5194 
5195 
5196 //*************************************************************************************************
5202 template< typename MT // Type of the dense matrix
5203  , size_t... CSAs > // Compile time submatrix arguments
5205 {
5206  using blaze::clear;
5207 
5208  for( size_t j=column(); j<column()+columns(); ++j )
5209  {
5210  const size_t ibegin( ( IsLower_v<MT> )
5211  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
5212  ?( max( j+1UL, row() ) )
5213  :( max( j, row() ) ) )
5214  :( row() ) );
5215  const size_t iend ( ( IsUpper_v<MT> )
5216  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
5217  ?( min( j, row()+rows() ) )
5218  :( min( j+1UL, row()+rows() ) ) )
5219  :( row()+rows() ) );
5220 
5221  for( size_t i=ibegin; i<iend; ++i )
5222  clear( matrix_(i,j) );
5223  }
5224 }
5226 //*************************************************************************************************
5227 
5228 
5229 //*************************************************************************************************
5236 template< typename MT // Type of the dense matrix
5237  , size_t... CSAs > // Compile time submatrix arguments
5239 {
5240  using blaze::clear;
5241 
5242  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
5243 
5244  const size_t ibegin( ( IsLower_v<MT> )
5245  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
5246  ?( max( j+1UL, row() ) )
5247  :( max( j, row() ) ) )
5248  :( row() ) );
5249  const size_t iend ( ( IsUpper_v<MT> )
5250  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
5251  ?( min( j, row()+rows() ) )
5252  :( min( j+1UL, row()+rows() ) ) )
5253  :( row()+rows() ) );
5254 
5255  for( size_t i=ibegin; i<iend; ++i )
5256  clear( matrix_(i,column()+j) );
5257 }
5259 //*************************************************************************************************
5260 
5261 
5262 //*************************************************************************************************
5272 template< typename MT // Type of the dense matrix
5273  , size_t... CSAs > // Compile time submatrix arguments
5274 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::hasOverlap() const noexcept
5275 {
5276  BLAZE_INTERNAL_ASSERT( IsSymmetric_v<MT> || IsHermitian_v<MT>, "Invalid matrix detected" );
5277 
5278  if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
5279  return false;
5280  else return true;
5281 }
5283 //*************************************************************************************************
5284 
5285 
5286 
5287 
5288 //=================================================================================================
5289 //
5290 // NUMERIC FUNCTIONS
5291 //
5292 //=================================================================================================
5293 
5294 //*************************************************************************************************
5312 template< typename MT // Type of the dense matrix
5313  , size_t... CSAs > // Compile time submatrix arguments
5314 inline Submatrix<MT,unaligned,true,true,CSAs...>&
5316 {
5317  if( rows() != columns() ) {
5318  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
5319  }
5320 
5321  if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
5322  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
5323  }
5324 
5325  decltype(auto) left( derestrict( *this ) );
5326  const ResultType tmp( trans( *this ) );
5327 
5328  smpAssign( left, tmp );
5329 
5330  return *this;
5331 }
5333 //*************************************************************************************************
5334 
5335 
5336 //*************************************************************************************************
5354 template< typename MT // Type of the dense matrix
5355  , size_t... CSAs > // Compile time submatrix arguments
5356 inline Submatrix<MT,unaligned,true,true,CSAs...>&
5357  Submatrix<MT,unaligned,true,true,CSAs...>::ctranspose()
5358 {
5359  if( rows() != columns() ) {
5360  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
5361  }
5362 
5363  if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
5364  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
5365  }
5366 
5367  decltype(auto) left( derestrict( *this ) );
5368  const ResultType tmp( ctrans( *this ) );
5369 
5370  smpAssign( left, tmp );
5371 
5372  return *this;
5373 }
5375 //*************************************************************************************************
5376 
5377 
5378 //*************************************************************************************************
5391 template< typename MT // Type of the dense matrix
5392  , size_t... CSAs > // Compile time submatrix arguments
5393 template< typename Other > // Data type of the scalar value
5394 inline Submatrix<MT,unaligned,true,true,CSAs...>&
5395  Submatrix<MT,unaligned,true,true,CSAs...>::scale( const Other& scalar )
5396 {
5398 
5399  const size_t jend( column() + columns() );
5400 
5401  for( size_t j=column(); j<jend; ++j )
5402  {
5403  const size_t ibegin( ( IsLower_v<MT> )
5404  ?( ( IsStrictlyLower_v<MT> )
5405  ?( max( j+1UL, row() ) )
5406  :( max( j, row() ) ) )
5407  :( row() ) );
5408  const size_t iend ( ( IsUpper_v<MT> )
5409  ?( ( IsStrictlyUpper_v<MT> )
5410  ?( min( j, row()+rows() ) )
5411  :( min( j+1UL, row()+rows() ) ) )
5412  :( row()+rows() ) );
5413 
5414  for( size_t i=ibegin; i<iend; ++i )
5415  matrix_(i,j) *= scalar;
5416  }
5417 
5418  return *this;
5419 }
5421 //*************************************************************************************************
5422 
5423 
5424 
5425 
5426 //=================================================================================================
5427 //
5428 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
5429 //
5430 //=================================================================================================
5431 
5432 //*************************************************************************************************
5443 template< typename MT // Type of the dense matrix
5444  , size_t... CSAs > // Compile time submatrix arguments
5445 template< typename Other > // Data type of the foreign expression
5446 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::canAlias( const Other* alias ) const noexcept
5447 {
5448  return matrix_.isAliased( alias );
5449 }
5451 //*************************************************************************************************
5452 
5453 
5454 //*************************************************************************************************
5465 template< typename MT // Type of the dense matrix
5466  , size_t... CSAs > // Compile time submatrix arguments
5467 template< typename MT2 // Data type of the foreign dense submatrix
5468  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
5469  , bool SO2 // Storage order of the foreign dense submatrix
5470  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
5471 inline bool
5472  Submatrix<MT,unaligned,true,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
5473 {
5474  return ( matrix_.isAliased( &alias->matrix_ ) &&
5475  ( row() + rows() > alias->row() ) &&
5476  ( row() < alias->row() + alias->rows() ) &&
5477  ( column() + columns() > alias->column() ) &&
5478  ( column() < alias->column() + alias->columns() ) );
5479 }
5481 //*************************************************************************************************
5482 
5483 
5484 //*************************************************************************************************
5495 template< typename MT // Type of the dense matrix
5496  , size_t... CSAs > // Compile time submatrix arguments
5497 template< typename Other > // Data type of the foreign expression
5498 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::isAliased( const Other* alias ) const noexcept
5499 {
5500  return matrix_.isAliased( alias );
5501 }
5503 //*************************************************************************************************
5504 
5505 
5506 //*************************************************************************************************
5517 template< typename MT // Type of the dense matrix
5518  , size_t... CSAs > // Compile time submatrix arguments
5519 template< typename MT2 // Data type of the foreign dense submatrix
5520  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
5521  , bool SO2 // Storage order of the foreign dense submatrix
5522  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
5523 inline bool
5524  Submatrix<MT,unaligned,true,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
5525 {
5526  return ( matrix_.isAliased( &alias->matrix_ ) &&
5527  ( row() + rows() > alias->row() ) &&
5528  ( row() < alias->row() + alias->rows() ) &&
5529  ( column() + columns() > alias->column() ) &&
5530  ( column() < alias->column() + alias->columns() ) );
5531 }
5533 //*************************************************************************************************
5534 
5535 
5536 //*************************************************************************************************
5546 template< typename MT // Type of the dense matrix
5547  , size_t... CSAs > // Compile time submatrix arguments
5548 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::isAligned() const noexcept
5549 {
5550  return isAligned_;
5551 }
5553 //*************************************************************************************************
5554 
5555 
5556 //*************************************************************************************************
5567 template< typename MT // Type of the dense matrix
5568  , size_t... CSAs > // Compile time submatrix arguments
5569 inline bool Submatrix<MT,unaligned,true,true,CSAs...>::canSMPAssign() const noexcept
5570 {
5571  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
5572 }
5574 //*************************************************************************************************
5575 
5576 
5577 //*************************************************************************************************
5592 template< typename MT // Type of the dense matrix
5593  , size_t... CSAs > // Compile time submatrix arguments
5594 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,true,true,CSAs...>::SIMDType
5595  Submatrix<MT,unaligned,true,true,CSAs...>::load( size_t i, size_t j ) const noexcept
5596 {
5597  if( isAligned_ )
5598  return loada( i, j );
5599  else
5600  return loadu( i, j );
5601 }
5603 //*************************************************************************************************
5604 
5605 
5606 //*************************************************************************************************
5621 template< typename MT // Type of the dense matrix
5622  , size_t... CSAs > // Compile time submatrix arguments
5623 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,true,true,CSAs...>::SIMDType
5624  Submatrix<MT,unaligned,true,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
5625 {
5627 
5628  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5629  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5630  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5631  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5632 
5633  return matrix_.loada( row()+i, column()+j );
5634 }
5636 //*************************************************************************************************
5637 
5638 
5639 //*************************************************************************************************
5654 template< typename MT // Type of the dense matrix
5655  , size_t... CSAs > // Compile time submatrix arguments
5656 BLAZE_ALWAYS_INLINE typename Submatrix<MT,unaligned,true,true,CSAs...>::SIMDType
5657  Submatrix<MT,unaligned,true,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
5658 {
5660 
5661  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5662  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5663  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5664  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5665 
5666  return matrix_.loadu( row()+i, column()+j );
5667 }
5669 //*************************************************************************************************
5670 
5671 
5672 //*************************************************************************************************
5688 template< typename MT // Type of the dense matrix
5689  , size_t... CSAs > // Compile time submatrix arguments
5691  Submatrix<MT,unaligned,true,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
5692 {
5693  if( isAligned_ )
5694  storea( i, j, value );
5695  else
5696  storeu( i, j, value );
5697 }
5699 //*************************************************************************************************
5700 
5701 
5702 //*************************************************************************************************
5718 template< typename MT // Type of the dense matrix
5719  , size_t... CSAs > // Compile time submatrix arguments
5721  Submatrix<MT,unaligned,true,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
5722 {
5724 
5725  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5726  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5727  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5728  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5729 
5730  matrix_.storea( row()+i, column()+j, value );
5731 }
5733 //*************************************************************************************************
5734 
5735 
5736 //*************************************************************************************************
5752 template< typename MT // Type of the dense matrix
5753  , size_t... CSAs > // Compile time submatrix arguments
5755  Submatrix<MT,unaligned,true,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
5756 {
5758 
5759  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5760  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5761  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5762  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5763 
5764  matrix_.storeu( row()+i, column()+j, value );
5765 }
5767 //*************************************************************************************************
5768 
5769 
5770 //*************************************************************************************************
5787 template< typename MT // Type of the dense matrix
5788  , size_t... CSAs > // Compile time submatrix arguments
5790  Submatrix<MT,unaligned,true,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
5791 {
5793 
5794  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
5795  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
5796  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
5797  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
5798 
5799  if( isAligned_ )
5800  matrix_.stream( row()+i, column()+j, value );
5801  else
5802  matrix_.storeu( row()+i, column()+j, value );
5803 }
5805 //*************************************************************************************************
5806 
5807 
5808 //*************************************************************************************************
5820 template< typename MT // Type of the dense matrix
5821  , size_t... CSAs > // Compile time submatrix arguments
5822 template< typename MT2 > // Type of the right-hand side dense matrix
5823 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
5824  -> DisableIf_t< VectorizedAssign_v<MT2> >
5825 {
5826  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
5827  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
5828 
5829  const size_t ipos( rows() & size_t(-2) );
5830  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
5831 
5832  for( size_t j=0UL; j<columns(); ++j ) {
5833  for( size_t i=0UL; i<ipos; i+=2UL ) {
5834  matrix_(row()+i ,column()+j) = (~rhs)(i ,j);
5835  matrix_(row()+i+1UL,column()+j) = (~rhs)(i+1UL,j);
5836  }
5837  if( ipos < rows() ) {
5838  matrix_(row()+ipos,column()+j) = (~rhs)(ipos,j);
5839  }
5840  }
5841 }
5843 //*************************************************************************************************
5844 
5845 
5846 //*************************************************************************************************
5858 template< typename MT // Type of the dense matrix
5859  , size_t... CSAs > // Compile time submatrix arguments
5860 template< typename MT2 > // Type of the right-hand side dense matrix
5861 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
5862  -> EnableIf_t< VectorizedAssign_v<MT2> >
5863 {
5865 
5866  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
5867  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
5868 
5869  const size_t ipos( rows() & size_t(-SIMDSIZE) );
5870  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
5871 
5872  if( useStreaming && isAligned_ &&
5873  rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
5874  !(~rhs).isAliased( &matrix_ ) )
5875  {
5876  for( size_t j=0UL; j<columns(); ++j )
5877  {
5878  size_t i( 0UL );
5879  Iterator left( begin(j) );
5880  ConstIterator_t<MT2> right( (~rhs).begin(j) );
5881 
5882  for( ; i<ipos; i+=SIMDSIZE ) {
5883  left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5884  }
5885  for( ; i<rows(); ++i ) {
5886  *left = *right; ++left; ++right;
5887  }
5888  }
5889  }
5890  else
5891  {
5892  for( size_t j=0UL; j<columns(); ++j )
5893  {
5894  size_t i( 0UL );
5895  Iterator left( begin(j) );
5896  ConstIterator_t<MT2> right( (~rhs).begin(j) );
5897 
5898  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
5899  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5900  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5901  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5902  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5903  }
5904  for( ; i<ipos; i+=SIMDSIZE ) {
5905  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
5906  }
5907  for( ; i<rows(); ++i ) {
5908  *left = *right; ++left; ++right;
5909  }
5910  }
5911  }
5912 }
5914 //*************************************************************************************************
5915 
5916 
5917 //*************************************************************************************************
5929 template< typename MT // Type of the dense matrix
5930  , size_t... CSAs > // Compile time submatrix arguments
5931 template< typename MT2 > // Type of the right-hand side dense matrix
5932 inline void Submatrix<MT,unaligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
5933 {
5935 
5936  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
5937  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
5938 
5939  constexpr size_t block( BLOCK_SIZE );
5940 
5941  for( size_t jj=0UL; jj<columns(); jj+=block ) {
5942  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
5943  for( size_t ii=0UL; ii<rows(); ii+=block ) {
5944  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
5945  for( size_t j=jj; j<jend; ++j ) {
5946  for( size_t i=ii; i<iend; ++i ) {
5947  matrix_(row()+i,column()+j) = (~rhs)(i,j);
5948  }
5949  }
5950  }
5951  }
5952 }
5954 //*************************************************************************************************
5955 
5956 
5957 //*************************************************************************************************
5969 template< typename MT // Type of the dense matrix
5970  , size_t... CSAs > // Compile time submatrix arguments
5971 template< typename MT2 > // Type of the right-hand side sparse matrix
5972 inline void Submatrix<MT,unaligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
5973 {
5974  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
5975  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
5976 
5977  for( size_t j=0UL; j<columns(); ++j )
5978  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
5979  matrix_(row()+element->index(),column()+j) = element->value();
5980 }
5982 //*************************************************************************************************
5983 
5984 
5985 //*************************************************************************************************
5997 template< typename MT // Type of the dense matrix
5998  , size_t... CSAs > // Compile time submatrix arguments
5999 template< typename MT2 > // Type of the right-hand side sparse matrix
6000 inline void Submatrix<MT,unaligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
6001 {
6003 
6004  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6005  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6006 
6007  for( size_t i=0UL; i<rows(); ++i )
6008  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
6009  matrix_(row()+i,column()+element->index()) = element->value();
6010 }
6012 //*************************************************************************************************
6013 
6014 
6015 //*************************************************************************************************
6027 template< typename MT // Type of the dense matrix
6028  , size_t... CSAs > // Compile time submatrix arguments
6029 template< typename MT2 > // Type of the right-hand side dense matrix
6030 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
6031  -> DisableIf_t< VectorizedAddAssign_v<MT2> >
6032 {
6033  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6034  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6035 
6036  const size_t ipos( rows() & size_t(-2) );
6037  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
6038 
6039  for( size_t j=0UL; j<columns(); ++j )
6040  {
6041  if( IsDiagonal_v<MT2> ) {
6042  matrix_(row()+j,column()+j) += (~rhs)(j,j);
6043  }
6044  else {
6045  for( size_t i=0UL; i<ipos; i+=2UL ) {
6046  matrix_(row()+i ,column()+j) += (~rhs)(i ,j);
6047  matrix_(row()+i+1UL,column()+j) += (~rhs)(i+1UL,j);
6048  }
6049  if( ipos < rows() ) {
6050  matrix_(row()+ipos,column()+j) += (~rhs)(ipos,j);
6051  }
6052  }
6053  }
6054 }
6056 //*************************************************************************************************
6057 
6058 
6059 //*************************************************************************************************
6071 template< typename MT // Type of the dense matrix
6072  , size_t... CSAs > // Compile time submatrix arguments
6073 template< typename MT2 > // Type of the right-hand side dense matrix
6074 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
6075  -> EnableIf_t< VectorizedAddAssign_v<MT2> >
6076 {
6078 
6079  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6080  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6081 
6082  for( size_t j=0UL; j<columns(); ++j )
6083  {
6084  const size_t ibegin( ( IsLower_v<MT> )
6085  ?( ( IsStrictlyLower_v<MT> ? j+1UL : j ) & size_t(-SIMDSIZE) )
6086  :( 0UL ) );
6087  const size_t iend ( ( IsUpper_v<MT> )
6088  ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
6089  :( rows() ) );
6090  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
6091 
6092  const size_t ipos( iend & size_t(-SIMDSIZE) );
6093  BLAZE_INTERNAL_ASSERT( ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
6094 
6095  size_t i( ibegin );
6096  Iterator left( begin(j) + ibegin );
6097  ConstIterator_t<MT2> right( (~rhs).begin(j) + ibegin );
6098 
6099  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
6100  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6101  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6102  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6103  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6104  }
6105  for( ; i<ipos; i+=SIMDSIZE ) {
6106  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6107  }
6108  for( ; i<iend; ++i ) {
6109  *left += *right; ++left; ++right;
6110  }
6111  }
6112 }
6114 //*************************************************************************************************
6115 
6116 
6117 //*************************************************************************************************
6129 template< typename MT // Type of the dense matrix
6130  , size_t... CSAs > // Compile time submatrix arguments
6131 template< typename MT2 > // Type of the right-hand side dense matrix
6132 inline void Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
6133 {
6135 
6136  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6137  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6138 
6139  constexpr size_t block( BLOCK_SIZE );
6140 
6141  for( size_t jj=0UL; jj<columns(); jj+=block ) {
6142  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
6143  for( size_t ii=0UL; ii<rows(); ii+=block ) {
6144  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
6145  for( size_t j=jj; j<jend; ++j ) {
6146  for( size_t i=ii; i<iend; ++i ) {
6147  matrix_(row()+i,column()+j) += (~rhs)(i,j);
6148  }
6149  }
6150  }
6151  }
6152 }
6154 //*************************************************************************************************
6155 
6156 
6157 //*************************************************************************************************
6169 template< typename MT // Type of the dense matrix
6170  , size_t... CSAs > // Compile time submatrix arguments
6171 template< typename MT2 > // Type of the right-hand side sparse matrix
6172 inline void Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
6173 {
6174  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6175  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6176 
6177  for( size_t j=0UL; j<columns(); ++j )
6178  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
6179  matrix_(row()+element->index(),column()+j) += element->value();
6180 }
6182 //*************************************************************************************************
6183 
6184 
6185 //*************************************************************************************************
6197 template< typename MT // Type of the dense matrix
6198  , size_t... CSAs > // Compile time submatrix arguments
6199 template< typename MT2 > // Type of the right-hand side sparse matrix
6200 inline void Submatrix<MT,unaligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
6201 {
6203 
6204  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6205  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6206 
6207  for( size_t i=0UL; i<rows(); ++i )
6208  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
6209  matrix_(row()+i,column()+element->index()) += element->value();
6210 }
6212 //*************************************************************************************************
6213 
6214 
6215 //*************************************************************************************************
6227 template< typename MT // Type of the dense matrix
6228  , size_t... CSAs > // Compile time submatrix arguments
6229 template< typename MT2 > // Type of the right-hand side dense matrix
6230 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
6231  -> DisableIf_t< VectorizedSubAssign_v<MT2> >
6232 {
6233  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6234  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6235 
6236  const size_t ipos( rows() & size_t(-2) );
6237  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
6238 
6239  for( size_t j=0UL; j<columns(); ++j )
6240  {
6241  if( IsDiagonal_v<MT2> ) {
6242  matrix_(row()+j,column()+j) -= (~rhs)(j,j);
6243  }
6244  else {
6245  for( size_t i=0UL; i<ipos; i+=2UL ) {
6246  matrix_(row()+i ,column()+j) -= (~rhs)(i ,j);
6247  matrix_(row()+i+1UL,column()+j) -= (~rhs)(i+1UL,j);
6248  }
6249  if( ipos < rows() ) {
6250  matrix_(row()+ipos,column()+j) -= (~rhs)(ipos,j);
6251  }
6252  }
6253  }
6254 }
6256 //*************************************************************************************************
6257 
6258 
6259 //*************************************************************************************************
6271 template< typename MT // Type of the dense matrix
6272  , size_t... CSAs > // Compile time submatrix arguments
6273 template< typename MT2 > // Type of the right-hand side dense matrix
6274 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
6275  -> EnableIf_t< VectorizedSubAssign_v<MT2> >
6276 {
6278 
6279  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6280  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6281 
6282  for( size_t j=0UL; j<columns(); ++j )
6283  {
6284  const size_t ibegin( ( IsLower_v<MT> )
6285  ?( ( IsStrictlyLower_v<MT> ? j+1UL : j ) & size_t(-SIMDSIZE) )
6286  :( 0UL ) );
6287  const size_t iend ( ( IsUpper_v<MT> )
6288  ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
6289  :( rows() ) );
6290  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
6291 
6292  const size_t ipos( iend & size_t(-SIMDSIZE) );
6293  BLAZE_INTERNAL_ASSERT( ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
6294 
6295  size_t i( ibegin );
6296  Iterator left( begin(j) + ibegin );
6297  ConstIterator_t<MT2> right( (~rhs).begin(j) + ibegin );
6298 
6299  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
6300  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6301  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6302  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6303  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6304  }
6305  for( ; i<ipos; i+=SIMDSIZE ) {
6306  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6307  }
6308  for( ; i<iend; ++i ) {
6309  *left -= *right; ++left; ++right;
6310  }
6311  }
6312 }
6314 //*************************************************************************************************
6315 
6316 
6317 //*************************************************************************************************
6329 template< typename MT // Type of the dense matrix
6330  , size_t... CSAs > // Compile time submatrix arguments
6331 template< typename MT2 > // Type of the right-hand side dense matrix
6332 inline void Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
6333 {
6335 
6336  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6337  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6338 
6339  constexpr size_t block( BLOCK_SIZE );
6340 
6341  for( size_t jj=0UL; jj<columns(); jj+=block ) {
6342  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
6343  for( size_t ii=0UL; ii<rows(); ii+=block ) {
6344  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
6345  for( size_t j=jj; j<jend; ++j ) {
6346  for( size_t i=ii; i<iend; ++i ) {
6347  matrix_(row()+i,column()+j) -= (~rhs)(i,j);
6348  }
6349  }
6350  }
6351  }
6352 }
6354 //*************************************************************************************************
6355 
6356 
6357 //*************************************************************************************************
6369 template< typename MT // Type of the dense matrix
6370  , size_t... CSAs > // Compile time submatrix arguments
6371 template< typename MT2 > // Type of the right-hand side sparse matrix
6372 inline void Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
6373 {
6374  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6375  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6376 
6377  for( size_t j=0UL; j<columns(); ++j )
6378  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
6379  matrix_(row()+element->index(),column()+j) -= element->value();
6380 }
6382 //*************************************************************************************************
6383 
6384 
6385 //*************************************************************************************************
6397 template< typename MT // Type of the dense matrix
6398  , size_t... CSAs > // Compile time submatrix arguments
6399 template< typename MT2 > // Type of the right-hand side sparse matrix
6400 inline void Submatrix<MT,unaligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
6401 {
6403 
6404  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6405  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6406 
6407  for( size_t i=0UL; i<rows(); ++i )
6408  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
6409  matrix_(row()+i,column()+element->index()) -= element->value();
6410 }
6412 //*************************************************************************************************
6413 
6414 
6415 //*************************************************************************************************
6427 template< typename MT // Type of the dense matrix
6428  , size_t... CSAs > // Compile time submatrix arguments
6429 template< typename MT2 > // Type of the right-hand side dense matrix
6430 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
6431  -> DisableIf_t< VectorizedSchurAssign_v<MT2> >
6432 {
6433  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6434  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6435 
6436  const size_t ipos( rows() & size_t(-2) );
6437  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
6438 
6439  for( size_t j=0UL; j<columns(); ++j ) {
6440  for( size_t i=0UL; i<ipos; i+=2UL ) {
6441  matrix_(row()+i ,column()+j) *= (~rhs)(i ,j);
6442  matrix_(row()+i+1UL,column()+j) *= (~rhs)(i+1UL,j);
6443  }
6444  if( ipos < rows() ) {
6445  matrix_(row()+ipos,column()+j) *= (~rhs)(ipos,j);
6446  }
6447  }
6448 }
6450 //*************************************************************************************************
6451 
6452 
6453 //*************************************************************************************************
6466 template< typename MT // Type of the dense matrix
6467  , size_t... CSAs > // Compile time submatrix arguments
6468 template< typename MT2 > // Type of the right-hand side dense matrix
6469 inline auto Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
6470  -> EnableIf_t< VectorizedSchurAssign_v<MT2> >
6471 {
6473 
6474  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6475  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6476 
6477  for( size_t j=0UL; j<columns(); ++j )
6478  {
6479  const size_t ipos( rows() & size_t(-SIMDSIZE) );
6480  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
6481 
6482  size_t i( 0UL );
6483  Iterator left( begin(j) );
6484  ConstIterator_t<MT2> right( (~rhs).begin(j) );
6485 
6486  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
6487  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6488  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6489  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6490  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6491  }
6492  for( ; i<ipos; i+=SIMDSIZE ) {
6493  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
6494  }
6495  for( ; i<rows(); ++i ) {
6496  *left *= *right; ++left; ++right;
6497  }
6498  }
6499 }
6501 //*************************************************************************************************
6502 
6503 
6504 //*************************************************************************************************
6516 template< typename MT // Type of the dense matrix
6517  , size_t... CSAs > // Compile time submatrix arguments
6518 template< typename MT2 > // Type of the right-hand side dense matrix
6519 inline void Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
6520 {
6522 
6523  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6524  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6525 
6526  constexpr size_t block( BLOCK_SIZE );
6527 
6528  for( size_t jj=0UL; jj<columns(); jj+=block ) {
6529  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
6530  for( size_t ii=0UL; ii<rows(); ii+=block ) {
6531  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
6532  for( size_t j=jj; j<jend; ++j ) {
6533  for( size_t i=ii; i<iend; ++i ) {
6534  matrix_(row()+i,column()+j) *= (~rhs)(i,j);
6535  }
6536  }
6537  }
6538  }
6539 }
6541 //*************************************************************************************************
6542 
6543 
6544 //*************************************************************************************************
6556 template< typename MT // Type of the dense matrix
6557  , size_t... CSAs > // Compile time submatrix arguments
6558 template< typename MT2 > // Type of the right-hand side sparse matrix
6559 inline void Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
6560 {
6561  using blaze::reset;
6562 
6563  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6564  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6565 
6566  for( size_t j=0UL; j<columns(); ++j )
6567  {
6568  size_t i( 0UL );
6569 
6570  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element ) {
6571  for( ; i<element->index(); ++i )
6572  reset( matrix_(row()+i,column()+j) );
6573  matrix_(row()+i,column()+j) *= element->value();
6574  ++i;
6575  }
6576 
6577  for( ; i<rows(); ++i ) {
6578  reset( matrix_(row()+i,column()+j) );
6579  }
6580  }
6581 }
6583 //*************************************************************************************************
6584 
6585 
6586 //*************************************************************************************************
6598 template< typename MT // Type of the dense matrix
6599  , size_t... CSAs > // Compile time submatrix arguments
6600 template< typename MT2 > // Type of the right-hand side sparse matrix
6601 inline void Submatrix<MT,unaligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
6602 {
6603  using blaze::reset;
6604 
6606 
6607  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
6608  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
6609 
6610  for( size_t i=0UL; i<rows(); ++i )
6611  {
6612  size_t j( 0UL );
6613 
6614  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element ) {
6615  for( ; j<element->index(); ++j )
6616  reset( matrix_(row()+i,column()+j) );
6617  matrix_(row()+i,column()+j) *= element->value();
6618  ++j;
6619  }
6620 
6621  for( ; j<columns(); ++j ) {
6622  reset( matrix_(row()+i,column()+j) );
6623  }
6624  }
6625 }
6627 //*************************************************************************************************
6628 
6629 
6630 
6631 
6632 
6633 
6634 
6635 
6636 //=================================================================================================
6637 //
6638 // CLASS TEMPLATE SPECIALIZATION FOR ALIGNED ROW-MAJOR DENSE SUBMATRICES
6639 //
6640 //=================================================================================================
6641 
6642 //*************************************************************************************************
6650 template< typename MT // Type of the dense matrix
6651  , size_t... CSAs > // Compile time submatrix arguments
6652 class Submatrix<MT,aligned,false,true,CSAs...>
6653  : public View< DenseMatrix< Submatrix<MT,aligned,false,true,CSAs...>, false > >
6654  , private SubmatrixData<CSAs...>
6655 {
6656  private:
6657  //**Type definitions****************************************************************************
6658  using DataType = SubmatrixData<CSAs...>;
6659  using Operand = If_t< IsExpression_v<MT>, MT, MT& >;
6660  //**********************************************************************************************
6661 
6662  //**********************************************************************************************
6664  template< typename MT1, typename MT2 >
6665  static constexpr bool EnforceEvaluation_v =
6666  ( IsRestricted_v<MT1> && RequiresEvaluation_v<MT2> );
6667  //**********************************************************************************************
6668 
6669  public:
6670  //**Type definitions****************************************************************************
6672  using This = Submatrix<MT,aligned,false,true,CSAs...>;
6673 
6674  using BaseType = DenseMatrix<This,false>;
6675  using ViewedType = MT;
6676  using ResultType = SubmatrixTrait_t<MT,CSAs...>;
6677  using OppositeType = OppositeType_t<ResultType>;
6678  using TransposeType = TransposeType_t<ResultType>;
6679  using ElementType = ElementType_t<MT>;
6680  using SIMDType = SIMDTrait_t<ElementType>;
6681  using ReturnType = ReturnType_t<MT>;
6682  using CompositeType = const Submatrix&;
6683 
6685  using ConstReference = ConstReference_t<MT>;
6686 
6688  using Reference = If_t< IsConst_v<MT>, ConstReference, Reference_t<MT> >;
6689 
6691  using ConstPointer = ConstPointer_t<MT>;
6692 
6694  using Pointer = If_t< IsConst_v<MT> || !HasMutableDataAccess_v<MT>, ConstPointer, Pointer_t<MT> >;
6695 
6697  using ConstIterator = ConstIterator_t<MT>;
6698 
6700  using Iterator = If_t< IsConst_v<MT>, ConstIterator, Iterator_t<MT> >;
6701  //**********************************************************************************************
6702 
6703  //**Compilation flags***************************************************************************
6705  static constexpr bool simdEnabled = MT::simdEnabled;
6706 
6708  static constexpr bool smpAssignable = MT::smpAssignable;
6709  //**********************************************************************************************
6710 
6711  //**Constructors********************************************************************************
6714  template< typename... RSAs >
6715  explicit inline Submatrix( MT& matrix, RSAs... args );
6716 
6717  Submatrix( const Submatrix& ) = default;
6719  //**********************************************************************************************
6720 
6721  //**Destructor**********************************************************************************
6724  ~Submatrix() = default;
6726  //**********************************************************************************************
6727 
6728  //**Data access functions***********************************************************************
6731  inline Reference operator()( size_t i, size_t j );
6732  inline ConstReference operator()( size_t i, size_t j ) const;
6733  inline Reference at( size_t i, size_t j );
6734  inline ConstReference at( size_t i, size_t j ) const;
6735  inline Pointer data () noexcept;
6736  inline ConstPointer data () const noexcept;
6737  inline Pointer data ( size_t i ) noexcept;
6738  inline ConstPointer data ( size_t i ) const noexcept;
6739  inline Iterator begin ( size_t i );
6740  inline ConstIterator begin ( size_t i ) const;
6741  inline ConstIterator cbegin( size_t i ) const;
6742  inline Iterator end ( size_t i );
6743  inline ConstIterator end ( size_t i ) const;
6744  inline ConstIterator cend ( size_t i ) const;
6746  //**********************************************************************************************
6747 
6748  //**Assignment operators************************************************************************
6751  inline Submatrix& operator=( const ElementType& rhs );
6752  inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
6753  inline Submatrix& operator=( const Submatrix& rhs );
6754 
6755  template< typename MT2, bool SO >
6756  inline Submatrix& operator=( const Matrix<MT2,SO>& rhs );
6757 
6758  template< typename MT2, bool SO >
6759  inline auto operator+=( const Matrix<MT2,SO>& rhs )
6760  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6761 
6762  template< typename MT2, bool SO >
6763  inline auto operator+=( const Matrix<MT2,SO>& rhs )
6764  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6765 
6766  template< typename MT2, bool SO >
6767  inline auto operator-=( const Matrix<MT2,SO>& rhs )
6768  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6769 
6770  template< typename MT2, bool SO >
6771  inline auto operator-=( const Matrix<MT2,SO>& rhs )
6772  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6773 
6774  template< typename MT2, bool SO >
6775  inline auto operator%=( const Matrix<MT2,SO>& rhs )
6776  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6777 
6778  template< typename MT2, bool SO >
6779  inline auto operator%=( const Matrix<MT2,SO>& rhs )
6780  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
6782  //**********************************************************************************************
6783 
6784  //**Utility functions***************************************************************************
6787  using DataType::row;
6788  using DataType::column;
6789  using DataType::rows;
6790  using DataType::columns;
6791 
6792  inline MT& operand() noexcept;
6793  inline const MT& operand() const noexcept;
6794 
6795  inline size_t spacing() const noexcept;
6796  inline size_t capacity() const noexcept;
6797  inline size_t capacity( size_t i ) const noexcept;
6798  inline size_t nonZeros() const;
6799  inline size_t nonZeros( size_t i ) const;
6800  inline void reset();
6801  inline void reset( size_t i );
6803  //**********************************************************************************************
6804 
6805  //**Numeric functions***************************************************************************
6808  inline Submatrix& transpose();
6809  inline Submatrix& ctranspose();
6810 
6811  template< typename Other > inline Submatrix& scale( const Other& scalar );
6813  //**********************************************************************************************
6814 
6815  private:
6816  //**********************************************************************************************
6818  template< typename MT2 >
6819  static constexpr bool VectorizedAssign_v =
6820  ( useOptimizedKernels &&
6821  simdEnabled && MT2::simdEnabled &&
6823  //**********************************************************************************************
6824 
6825  //**********************************************************************************************
6827  template< typename MT2 >
6828  static constexpr bool VectorizedAddAssign_v =
6829  ( useOptimizedKernels &&
6830  simdEnabled && MT2::simdEnabled &&
6833  !IsDiagonal_v<MT2> );
6834  //**********************************************************************************************
6835 
6836  //**********************************************************************************************
6838  template< typename MT2 >
6839  static constexpr bool VectorizedSubAssign_v =
6840  ( useOptimizedKernels &&
6841  simdEnabled && MT2::simdEnabled &&
6844  !IsDiagonal_v<MT2> );
6845  //**********************************************************************************************
6846 
6847  //**********************************************************************************************
6849  template< typename MT2 >
6850  static constexpr bool VectorizedSchurAssign_v =
6851  ( useOptimizedKernels &&
6852  simdEnabled && MT2::simdEnabled &&
6855  //**********************************************************************************************
6856 
6857  //**SIMD properties*****************************************************************************
6859  static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
6860  //**********************************************************************************************
6861 
6862  public:
6863  //**Expression template evaluation functions****************************************************
6866  template< typename Other >
6867  inline bool canAlias( const Other* alias ) const noexcept;
6868 
6869  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
6870  inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
6871 
6872  template< typename Other >
6873  inline bool isAliased( const Other* alias ) const noexcept;
6874 
6875  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
6876  inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
6877 
6878  inline bool isAligned () const noexcept;
6879  inline bool canSMPAssign() const noexcept;
6880 
6881  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
6882  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
6883  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
6884 
6885  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
6886  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
6887  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
6888  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
6889 
6890  template< typename MT2 >
6891  inline auto assign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT2> >;
6892 
6893  template< typename MT2 >
6894  inline auto assign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT2> >;
6895 
6896  template< typename MT2 > inline void assign( const DenseMatrix<MT2,true>& rhs );
6897  template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
6898  template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
6899 
6900  template< typename MT2 >
6901  inline auto addAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT2> >;
6902 
6903  template< typename MT2 >
6904  inline auto addAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT2> >;
6905 
6906  template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,true>& rhs );
6907  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
6908  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
6909 
6910  template< typename MT2 >
6911  inline auto subAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT2> >;
6912 
6913  template< typename MT2 >
6914  inline auto subAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT2> >;
6915 
6916  template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,true>& rhs );
6917  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
6918  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
6919 
6920  template< typename MT2 >
6921  inline auto schurAssign( const DenseMatrix<MT2,false>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT2> >;
6922 
6923  template< typename MT2 >
6924  inline auto schurAssign( const DenseMatrix<MT2,false>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT2> >;
6925 
6926  template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,true>& rhs );
6927  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
6928  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
6930  //**********************************************************************************************
6931 
6932  private:
6933  //**Utility functions***************************************************************************
6936  inline bool hasOverlap() const noexcept;
6938  //**********************************************************************************************
6939 
6940  //**Member variables****************************************************************************
6943  Operand matrix_;
6944 
6945  //**********************************************************************************************
6946 
6947  //**Friend declarations*************************************************************************
6948  template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
6949  //**********************************************************************************************
6950 
6951  //**Compile time checks*************************************************************************
6959  //**********************************************************************************************
6960 };
6962 //*************************************************************************************************
6963 
6964 
6965 
6966 
6967 //=================================================================================================
6968 //
6969 // CONSTRUCTORS
6970 //
6971 //=================================================================================================
6972 
6973 //*************************************************************************************************
6986 template< typename MT // Type of the dense matrix
6987  , size_t... CSAs > // Compile time submatrix arguments
6988 template< typename... RSAs > // Runtime submatrix arguments
6989 inline Submatrix<MT,aligned,false,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
6990  : DataType( args... ) // Base class initialization
6991  , matrix_ ( matrix ) // The matrix containing the submatrix
6992 {
6993  if( !Contains_v< TypeList<RSAs...>, Unchecked > )
6994  {
6995  if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
6996  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
6997  }
6998 
6999  if( simdEnabled && IsContiguous_v<MT> &&
7000  ( !checkAlignment( data() ) ||
7001  ( rows() > 1UL && matrix_.spacing() % SIMDSIZE != 0UL ) ) ) {
7002  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix alignment" );
7003  }
7004  }
7005  else
7006  {
7007  BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
7008  BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
7009 
7010  BLAZE_USER_ASSERT( !simdEnabled || !IsContiguous_v<MT> || checkAlignment( data() ), "Invalid submatrix alignment" );
7011  BLAZE_USER_ASSERT( !simdEnabled || !IsContiguous_v<MT> || rows() <= 1UL || matrix_.spacing() % SIMDSIZE == 0UL, "Invalid submatrix alignment" );
7012  }
7013 }
7015 //*************************************************************************************************
7016 
7017 
7018 
7019 
7020 //=================================================================================================
7021 //
7022 // DATA ACCESS FUNCTIONS
7023 //
7024 //=================================================================================================
7025 
7026 //*************************************************************************************************
7037 template< typename MT // Type of the dense matrix
7038  , size_t... CSAs > // Compile time submatrix arguments
7039 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Reference
7040  Submatrix<MT,aligned,false,true,CSAs...>::operator()( size_t i, size_t j )
7041 {
7042  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
7043  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
7044 
7045  return matrix_(row()+i,column()+j);
7046 }
7048 //*************************************************************************************************
7049 
7050 
7051 //*************************************************************************************************
7062 template< typename MT // Type of the dense matrix
7063  , size_t... CSAs > // Compile time submatrix arguments
7064 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstReference
7065  Submatrix<MT,aligned,false,true,CSAs...>::operator()( size_t i, size_t j ) const
7066 {
7067  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
7068  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
7069 
7070  return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
7071 }
7073 //*************************************************************************************************
7074 
7075 
7076 //*************************************************************************************************
7088 template< typename MT // Type of the dense matrix
7089  , size_t... CSAs > // Compile time submatrix arguments
7090 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Reference
7091  Submatrix<MT,aligned,false,true,CSAs...>::at( size_t i, size_t j )
7092 {
7093  if( i >= rows() ) {
7094  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
7095  }
7096  if( j >= columns() ) {
7097  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
7098  }
7099  return (*this)(i,j);
7100 }
7102 //*************************************************************************************************
7103 
7104 
7105 //*************************************************************************************************
7117 template< typename MT // Type of the dense matrix
7118  , size_t... CSAs > // Compile time submatrix arguments
7119 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstReference
7120  Submatrix<MT,aligned,false,true,CSAs...>::at( size_t i, size_t j ) const
7121 {
7122  if( i >= rows() ) {
7123  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
7124  }
7125  if( j >= columns() ) {
7126  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
7127  }
7128  return (*this)(i,j);
7129 }
7131 //*************************************************************************************************
7132 
7133 
7134 //*************************************************************************************************
7144 template< typename MT // Type of the dense matrix
7145  , size_t... CSAs > // Compile time submatrix arguments
7146 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Pointer
7148 {
7149  return matrix_.data() + row()*spacing() + column();
7150 }
7152 //*************************************************************************************************
7153 
7154 
7155 //*************************************************************************************************
7165 template< typename MT // Type of the dense matrix
7166  , size_t... CSAs > // Compile time submatrix arguments
7167 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstPointer
7169 {
7170  return matrix_.data() + row()*spacing() + column();
7171 }
7173 //*************************************************************************************************
7174 
7175 
7176 //*************************************************************************************************
7185 template< typename MT // Type of the dense matrix
7186  , size_t... CSAs > // Compile time submatrix arguments
7187 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Pointer
7189 {
7190  return matrix_.data() + (row()+i)*spacing() + column();
7191 }
7193 //*************************************************************************************************
7194 
7195 
7196 //*************************************************************************************************
7205 template< typename MT // Type of the dense matrix
7206  , size_t... CSAs > // Compile time submatrix arguments
7207 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstPointer
7208  Submatrix<MT,aligned,false,true,CSAs...>::data( size_t i ) const noexcept
7209 {
7210  return matrix_.data() + (row()+i)*spacing() + column();
7211 }
7213 //*************************************************************************************************
7214 
7215 
7216 //*************************************************************************************************
7228 template< typename MT // Type of the dense matrix
7229  , size_t... CSAs > // Compile time submatrix arguments
7230 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Iterator
7232 {
7233  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7234  return ( matrix_.begin( row() + i ) + column() );
7235 }
7237 //*************************************************************************************************
7238 
7239 
7240 //*************************************************************************************************
7252 template< typename MT // Type of the dense matrix
7253  , size_t... CSAs > // Compile time submatrix arguments
7254 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7256 {
7257  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7258  return ( matrix_.cbegin( row() + i ) + column() );
7259 }
7261 //*************************************************************************************************
7262 
7263 
7264 //*************************************************************************************************
7276 template< typename MT // Type of the dense matrix
7277  , size_t... CSAs > // Compile time submatrix arguments
7278 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7280 {
7281  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7282  return ( matrix_.cbegin( row() + i ) + column() );
7283 }
7285 //*************************************************************************************************
7286 
7287 
7288 //*************************************************************************************************
7300 template< typename MT // Type of the dense matrix
7301  , size_t... CSAs > // Compile time submatrix arguments
7302 inline typename Submatrix<MT,aligned,false,true,CSAs...>::Iterator
7304 {
7305  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7306  return ( matrix_.begin( row() + i ) + column() + columns() );
7307 }
7309 //*************************************************************************************************
7310 
7311 
7312 //*************************************************************************************************
7324 template< typename MT // Type of the dense matrix
7325  , size_t... CSAs > // Compile time submatrix arguments
7326 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7328 {
7329  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7330  return ( matrix_.cbegin( row() + i ) + column() + columns() );
7331 }
7333 //*************************************************************************************************
7334 
7335 
7336 //*************************************************************************************************
7348 template< typename MT // Type of the dense matrix
7349  , size_t... CSAs > // Compile time submatrix arguments
7350 inline typename Submatrix<MT,aligned,false,true,CSAs...>::ConstIterator
7352 {
7353  BLAZE_USER_ASSERT( i < rows(), "Invalid dense submatrix row access index" );
7354  return ( matrix_.cbegin( row() + i ) + column() + columns() );
7355 }
7357 //*************************************************************************************************
7358 
7359 
7360 
7361 
7362 //=================================================================================================
7363 //
7364 // ASSIGNMENT OPERATORS
7365 //
7366 //=================================================================================================
7367 
7368 //*************************************************************************************************
7379 template< typename MT // Type of the dense matrix
7380  , size_t... CSAs > // Compile time submatrix arguments
7381 inline Submatrix<MT,aligned,false,true,CSAs...>&
7382  Submatrix<MT,aligned,false,true,CSAs...>::operator=( const ElementType& rhs )
7383 {
7384  const size_t iend( row() + rows() );
7385  decltype(auto) left( derestrict( matrix_ ) );
7386 
7387  for( size_t i=row(); i<iend; ++i )
7388  {
7389  const size_t jbegin( ( IsUpper_v<MT> )
7390  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
7391  ?( max( i+1UL, column() ) )
7392  :( max( i, column() ) ) )
7393  :( column() ) );
7394  const size_t jend ( ( IsLower_v<MT> )
7395  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
7396  ?( min( i, column()+columns() ) )
7397  :( min( i+1UL, column()+columns() ) ) )
7398  :( column()+columns() ) );
7399 
7400  for( size_t j=jbegin; j<jend; ++j ) {
7401  if( !IsRestricted_v<MT> || IsTriangular_v<MT> || trySet( matrix_, i, j, rhs ) )
7402  left(i,j) = rhs;
7403  }
7404  }
7405 
7406  return *this;
7407 }
7409 //*************************************************************************************************
7410 
7411 
7412 //*************************************************************************************************
7428 template< typename MT // Type of the dense matrix
7429  , size_t... CSAs > // Compile time submatrix arguments
7430 inline Submatrix<MT,aligned,false,true,CSAs...>&
7431  Submatrix<MT,aligned,false,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
7432 {
7433  if( list.size() != rows() ) {
7434  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
7435  }
7436 
7437  if( IsRestricted_v<MT> ) {
7438  const InitializerMatrix<ElementType> tmp( list, columns() );
7439  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7440  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7441  }
7442  }
7443 
7444  decltype(auto) left( derestrict( *this ) );
7445  size_t i( 0UL );
7446 
7447  for( const auto& rowList : list ) {
7448  std::fill( std::copy( rowList.begin(), rowList.end(), left.begin(i) ), left.end(i), ElementType() );
7449  ++i;
7450  }
7451 
7452  return *this;
7453 }
7455 //*************************************************************************************************
7456 
7457 
7458 //*************************************************************************************************
7473 template< typename MT // Type of the dense matrix
7474  , size_t... CSAs > // Compile time submatrix arguments
7475 inline Submatrix<MT,aligned,false,true,CSAs...>&
7476  Submatrix<MT,aligned,false,true,CSAs...>::operator=( const Submatrix& rhs )
7477 {
7480 
7481  if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
7482  return *this;
7483 
7484  if( rows() != rhs.rows() || columns() != rhs.columns() ) {
7485  BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
7486  }
7487 
7488  if( !tryAssign( matrix_, rhs, row(), column() ) ) {
7489  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7490  }
7491 
7492  decltype(auto) left( derestrict( *this ) );
7493 
7494  if( rhs.canAlias( &matrix_ ) ) {
7495  const ResultType tmp( rhs );
7496  smpAssign( left, tmp );
7497  }
7498  else {
7499  smpAssign( left, rhs );
7500  }
7501 
7502  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7503 
7504  return *this;
7505 }
7507 //*************************************************************************************************
7508 
7509 
7510 //*************************************************************************************************
7525 template< typename MT // Type of the dense matrix
7526  , size_t... CSAs > // Compile time submatrix arguments
7527 template< typename MT2 // Type of the right-hand side matrix
7528  , bool SO > // Storage order of the right-hand side matrix
7529 inline Submatrix<MT,aligned,false,true,CSAs...>&
7530  Submatrix<MT,aligned,false,true,CSAs...>::operator=( const Matrix<MT2,SO>& rhs )
7531 {
7532  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7533 
7534  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7535  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7536  }
7537 
7538  using Right = If_t< IsRestricted_v<MT>, CompositeType_t<MT2>, const MT2& >;
7539  Right right( ~rhs );
7540 
7541  if( !tryAssign( matrix_, right, row(), column() ) ) {
7542  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7543  }
7544 
7545  decltype(auto) left( derestrict( *this ) );
7546 
7547  if( IsReference_v<Right> && right.canAlias( &matrix_ ) ) {
7548  const ResultType_t<MT2> tmp( right );
7549  if( IsSparseMatrix_v<MT2> )
7550  reset();
7551  smpAssign( left, tmp );
7552  }
7553  else {
7554  if( IsSparseMatrix_v<MT2> )
7555  reset();
7556  smpAssign( left, right );
7557  }
7558 
7559  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7560 
7561  return *this;
7562 }
7564 //*************************************************************************************************
7565 
7566 
7567 //*************************************************************************************************
7581 template< typename MT // Type of the dense matrix
7582  , size_t... CSAs > // Compile time submatrix arguments
7583 template< typename MT2 // Type of the right-hand side matrix
7584  , bool SO > // Storage order of the right-hand side matrix
7585 inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
7586  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7587 {
7590  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7591 
7592  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
7593 
7596 
7597  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7598  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7599  }
7600 
7601  if( !tryAddAssign( matrix_, ~rhs, row(), column() ) ) {
7602  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7603  }
7604 
7605  decltype(auto) left( derestrict( *this ) );
7606 
7607  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
7608  (~rhs).canAlias( &matrix_ ) ) {
7609  const AddType tmp( *this + (~rhs) );
7610  smpAssign( left, tmp );
7611  }
7612  else {
7613  smpAddAssign( left, ~rhs );
7614  }
7615 
7616  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7617 
7618  return *this;
7619 }
7621 //*************************************************************************************************
7622 
7623 
7624 //*************************************************************************************************
7638 template< typename MT // Type of the dense matrix
7639  , size_t... CSAs > // Compile time submatrix arguments
7640 template< typename MT2 // Type of the right-hand side matrix
7641  , bool SO > // Storage order of the right-hand side matrix
7642 inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
7643  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7644 {
7647  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7648 
7649  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
7650 
7653 
7654  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7655  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7656  }
7657 
7658  const AddType tmp( *this + (~rhs) );
7659 
7660  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7661  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7662  }
7663 
7664  decltype(auto) left( derestrict( *this ) );
7665 
7666  smpAssign( left, tmp );
7667 
7668  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7669 
7670  return *this;
7671 }
7673 //*************************************************************************************************
7674 
7675 
7676 //*************************************************************************************************
7690 template< typename MT // Type of the dense matrix
7691  , size_t... CSAs > // Compile time submatrix arguments
7692 template< typename MT2 // Type of the right-hand side matrix
7693  , bool SO > // Storage order of the right-hand side matrix
7694 inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
7695  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7696 {
7699  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7700 
7701  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
7702 
7705 
7706  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7707  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7708  }
7709 
7710  if( !trySubAssign( matrix_, ~rhs, row(), column() ) ) {
7711  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7712  }
7713 
7714  decltype(auto) left( derestrict( *this ) );
7715 
7716  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
7717  (~rhs).canAlias( &matrix_ ) ) {
7718  const SubType tmp( *this - (~rhs ) );
7719  smpAssign( left, tmp );
7720  }
7721  else {
7722  smpSubAssign( left, ~rhs );
7723  }
7724 
7725  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7726 
7727  return *this;
7728 }
7730 //*************************************************************************************************
7731 
7732 
7733 //*************************************************************************************************
7747 template< typename MT // Type of the dense matrix
7748  , size_t... CSAs > // Compile time submatrix arguments
7749 template< typename MT2 // Type of the right-hand side matrix
7750  , bool SO > // Storage order of the right-hand side matrix
7751 inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
7752  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7753 {
7756  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7757 
7758  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
7759 
7762 
7763  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7764  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7765  }
7766 
7767  const SubType tmp( *this - (~rhs) );
7768 
7769  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7770  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7771  }
7772 
7773  decltype(auto) left( derestrict( *this ) );
7774 
7775  smpAssign( left, tmp );
7776 
7777  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7778 
7779  return *this;
7780 }
7782 //*************************************************************************************************
7783 
7784 
7785 //*************************************************************************************************
7799 template< typename MT // Type of the dense matrix
7800  , size_t... CSAs > // Compile time submatrix arguments
7801 template< typename MT2 // Type of the right-hand side matrix
7802  , bool SO > // Storage order of the right-hand side matrix
7803 inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
7804  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7805 {
7808  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7809 
7810  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
7811 
7813 
7814  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7815  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7816  }
7817 
7818  if( !trySchurAssign( matrix_, ~rhs, row(), column() ) ) {
7819  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7820  }
7821 
7822  decltype(auto) left( derestrict( *this ) );
7823 
7824  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
7825  (~rhs).canAlias( &matrix_ ) ) {
7826  const SchurType tmp( *this % (~rhs) );
7827  if( IsSparseMatrix_v<SchurType> )
7828  reset();
7829  smpAssign( left, tmp );
7830  }
7831  else {
7832  smpSchurAssign( left, ~rhs );
7833  }
7834 
7835  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7836 
7837  return *this;
7838 }
7840 //*************************************************************************************************
7841 
7842 
7843 //*************************************************************************************************
7857 template< typename MT // Type of the dense matrix
7858  , size_t... CSAs > // Compile time submatrix arguments
7859 template< typename MT2 // Type of the right-hand side matrix
7860  , bool SO > // Storage order of the right-hand side matrix
7861 inline auto Submatrix<MT,aligned,false,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
7862  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
7863 {
7866  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
7867 
7868  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
7869 
7871 
7872  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
7873  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
7874  }
7875 
7876  const SchurType tmp( *this % (~rhs) );
7877 
7878  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
7879  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
7880  }
7881 
7882  decltype(auto) left( derestrict( *this ) );
7883 
7884  if( IsSparseMatrix_v<SchurType> ) {
7885  reset();
7886  }
7887 
7888  smpAssign( left, tmp );
7889 
7890  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
7891 
7892  return *this;
7893 }
7895 //*************************************************************************************************
7896 
7897 
7898 
7899 
7900 //=================================================================================================
7901 //
7902 // UTILITY FUNCTIONS
7903 //
7904 //=================================================================================================
7905 
7906 //*************************************************************************************************
7912 template< typename MT // Type of the dense matrix
7913  , size_t... CSAs > // Compile time submatrix arguments
7914 inline MT& Submatrix<MT,aligned,false,true,CSAs...>::operand() noexcept
7915 {
7916  return matrix_;
7917 }
7919 //*************************************************************************************************
7920 
7921 
7922 //*************************************************************************************************
7928 template< typename MT // Type of the dense matrix
7929  , size_t... CSAs > // Compile time submatrix arguments
7930 inline const MT& Submatrix<MT,aligned,false,true,CSAs...>::operand() const noexcept
7931 {
7932  return matrix_;
7933 }
7935 //*************************************************************************************************
7936 
7937 
7938 //*************************************************************************************************
7949 template< typename MT // Type of the dense matrix
7950  , size_t... CSAs > // Compile time submatrix arguments
7951 inline size_t Submatrix<MT,aligned,false,true,CSAs...>::spacing() const noexcept
7952 {
7953  return matrix_.spacing();
7954 }
7956 //*************************************************************************************************
7957 
7958 
7959 //*************************************************************************************************
7965 template< typename MT // Type of the dense matrix
7966  , size_t... CSAs > // Compile time submatrix arguments
7967 inline size_t Submatrix<MT,aligned,false,true,CSAs...>::capacity() const noexcept
7968 {
7969  return rows() * columns();
7970 }
7972 //*************************************************************************************************
7973 
7974 
7975 //*************************************************************************************************
7987 template< typename MT // Type of the dense matrix
7988  , size_t... CSAs > // Compile time submatrix arguments
7989 inline size_t Submatrix<MT,aligned,false,true,CSAs...>::capacity( size_t i ) const noexcept
7990 {
7991  UNUSED_PARAMETER( i );
7992 
7993  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
7994 
7995  return columns();
7996 }
7998 //*************************************************************************************************
7999 
8000 
8001 //*************************************************************************************************
8007 template< typename MT // Type of the dense matrix
8008  , size_t... CSAs > // Compile time submatrix arguments
8010 {
8011  const size_t iend( row() + rows() );
8012  const size_t jend( column() + columns() );
8013  size_t nonzeros( 0UL );
8014 
8015  for( size_t i=row(); i<iend; ++i )
8016  for( size_t j=column(); j<jend; ++j )
8017  if( !isDefault( matrix_(i,j) ) )
8018  ++nonzeros;
8019 
8020  return nonzeros;
8021 }
8023 //*************************************************************************************************
8024 
8025 
8026 //*************************************************************************************************
8038 template< typename MT // Type of the dense matrix
8039  , size_t... CSAs > // Compile time submatrix arguments
8040 inline size_t Submatrix<MT,aligned,false,true,CSAs...>::nonZeros( size_t i ) const
8041 {
8042  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
8043 
8044  const size_t jend( column() + columns() );
8045  size_t nonzeros( 0UL );
8046 
8047  for( size_t j=column(); j<jend; ++j )
8048  if( !isDefault( matrix_(row()+i,j) ) )
8049  ++nonzeros;
8050 
8051  return nonzeros;
8052 }
8054 //*************************************************************************************************
8055 
8056 
8057 //*************************************************************************************************
8063 template< typename MT // Type of the dense matrix
8064  , size_t... CSAs > // Compile time submatrix arguments
8066 {
8067  using blaze::clear;
8068 
8069  for( size_t i=row(); i<row()+rows(); ++i )
8070  {
8071  const size_t jbegin( ( IsUpper_v<MT> )
8072  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
8073  ?( max( i+1UL, column() ) )
8074  :( max( i, column() ) ) )
8075  :( column() ) );
8076  const size_t jend ( ( IsLower_v<MT> )
8077  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
8078  ?( min( i, column()+columns() ) )
8079  :( min( i+1UL, column()+columns() ) ) )
8080  :( column()+columns() ) );
8081 
8082  for( size_t j=jbegin; j<jend; ++j )
8083  clear( matrix_(i,j) );
8084  }
8085 }
8087 //*************************************************************************************************
8088 
8089 
8090 //*************************************************************************************************
8102 template< typename MT // Type of the dense matrix
8103  , size_t... CSAs > // Compile time submatrix arguments
8105 {
8106  using blaze::clear;
8107 
8108  BLAZE_USER_ASSERT( i < rows(), "Invalid row access index" );
8109 
8110  const size_t jbegin( ( IsUpper_v<MT> )
8111  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
8112  ?( max( i+1UL, column() ) )
8113  :( max( i, column() ) ) )
8114  :( column() ) );
8115  const size_t jend ( ( IsLower_v<MT> )
8116  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
8117  ?( min( i, column()+columns() ) )
8118  :( min( i+1UL, column()+columns() ) ) )
8119  :( column()+columns() ) );
8120 
8121  for( size_t j=jbegin; j<jend; ++j )
8122  clear( matrix_(row()+i,j) );
8123 }
8125 //*************************************************************************************************
8126 
8127 
8128 //*************************************************************************************************
8138 template< typename MT // Type of the dense matrix
8139  , size_t... CSAs > // Compile time submatrix arguments
8140 inline bool Submatrix<MT,aligned,false,true,CSAs...>::hasOverlap() const noexcept
8141 {
8142  BLAZE_INTERNAL_ASSERT( IsSymmetric_v<MT> || IsHermitian_v<MT>, "Invalid matrix detected" );
8143 
8144  if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
8145  return false;
8146  else return true;
8147 }
8149 //*************************************************************************************************
8150 
8151 
8152 
8153 
8154 //=================================================================================================
8155 //
8156 // NUMERIC FUNCTIONS
8157 //
8158 //=================================================================================================
8159 
8160 //*************************************************************************************************
8178 template< typename MT // Type of the dense matrix
8179  , size_t... CSAs > // Compile time submatrix arguments
8180 inline Submatrix<MT,aligned,false,true,CSAs...>&
8182 {
8183  if( rows() != columns() ) {
8184  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
8185  }
8186 
8187  if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
8188  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
8189  }
8190 
8191  decltype(auto) left( derestrict( *this ) );
8192  const ResultType tmp( trans( *this ) );
8193 
8194  smpAssign( left, tmp );
8195 
8196  return *this;
8197 }
8199 //*************************************************************************************************
8200 
8201 
8202 //*************************************************************************************************
8220 template< typename MT // Type of the dense matrix
8221  , size_t... CSAs > // Compile time submatrix arguments
8222 inline Submatrix<MT,aligned,false,true,CSAs...>&
8223  Submatrix<MT,aligned,false,true,CSAs...>::ctranspose()
8224 {
8225  if( rows() != columns() ) {
8226  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
8227  }
8228 
8229  if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
8230  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
8231  }
8232 
8233  decltype(auto) left( derestrict( *this ) );
8234  const ResultType tmp( ctrans( *this ) );
8235 
8236  smpAssign( left, tmp );
8237 
8238  return *this;
8239 }
8241 //*************************************************************************************************
8242 
8243 
8244 //*************************************************************************************************
8257 template< typename MT // Type of the dense matrix
8258  , size_t... CSAs > // Compile time submatrix arguments
8259 template< typename Other > // Data type of the scalar value
8260 inline Submatrix<MT,aligned,false,true,CSAs...>&
8261  Submatrix<MT,aligned,false,true,CSAs...>::scale( const Other& scalar )
8262 {
8264 
8265  const size_t iend( row() + rows() );
8266 
8267  for( size_t i=row(); i<iend; ++i )
8268  {
8269  const size_t jbegin( ( IsUpper_v<MT> )
8270  ?( ( IsStrictlyUpper_v<MT> )
8271  ?( max( i+1UL, column() ) )
8272  :( max( i, column() ) ) )
8273  :( column() ) );
8274  const size_t jend ( ( IsLower_v<MT> )
8275  ?( ( IsStrictlyLower_v<MT> )
8276  ?( min( i, column()+columns() ) )
8277  :( min( i+1UL, column()+columns() ) ) )
8278  :( column()+columns() ) );
8279 
8280  for( size_t j=jbegin; j<jend; ++j )
8281  matrix_(i,j) *= scalar;
8282  }
8283 
8284  return *this;
8285 }
8287 //*************************************************************************************************
8288 
8289 
8290 
8291 
8292 //=================================================================================================
8293 //
8294 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
8295 //
8296 //=================================================================================================
8297 
8298 //*************************************************************************************************
8309 template< typename MT // Type of the dense matrix
8310  , size_t... CSAs > // Compile time submatrix arguments
8311 template< typename Other > // Data type of the foreign expression
8312 inline bool Submatrix<MT,aligned,false,true,CSAs...>::canAlias( const Other* alias ) const noexcept
8313 {
8314  return matrix_.isAliased( alias );
8315 }
8317 //*************************************************************************************************
8318 
8319 
8320 //*************************************************************************************************
8331 template< typename MT // Type of the dense matrix
8332  , size_t... CSAs > // Compile time submatrix arguments
8333 template< typename MT2 // Data type of the foreign dense submatrix
8334  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
8335  , bool SO2 // Storage order of the foreign dense submatrix
8336  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
8337 inline bool
8338  Submatrix<MT,aligned,false,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
8339 {
8340  return ( matrix_.isAliased( &alias->matrix_ ) &&
8341  ( row() + rows() > alias->row() ) &&
8342  ( row() < alias->row() + alias->rows() ) &&
8343  ( column() + columns() > alias->column() ) &&
8344  ( column() < alias->column() + alias->columns() ) );
8345 }
8347 //*************************************************************************************************
8348 
8349 
8350 //*************************************************************************************************
8361 template< typename MT // Type of the dense matrix
8362  , size_t... CSAs > // Compile time submatrix arguments
8363 template< typename Other > // Data type of the foreign expression
8364 inline bool Submatrix<MT,aligned,false,true,CSAs...>::isAliased( const Other* alias ) const noexcept
8365 {
8366  return matrix_.isAliased( alias );
8367 }
8369 //*************************************************************************************************
8370 
8371 
8372 //*************************************************************************************************
8383 template< typename MT // Type of the dense matrix
8384  , size_t... CSAs > // Compile time submatrix arguments
8385 template< typename MT2 // Data type of the foreign dense submatrix
8386  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
8387  , bool SO2 // Storage order of the foreign dense submatrix
8388  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
8389 inline bool
8390  Submatrix<MT,aligned,false,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
8391 {
8392  return ( matrix_.isAliased( &alias->matrix_ ) &&
8393  ( row() + rows() > alias->row() ) &&
8394  ( row() < alias->row() + alias->rows() ) &&
8395  ( column() + columns() > alias->column() ) &&
8396  ( column() < alias->column() + alias->columns() ) );
8397 }
8399 //*************************************************************************************************
8400 
8401 
8402 //*************************************************************************************************
8412 template< typename MT // Type of the dense matrix
8413  , size_t... CSAs > // Compile time submatrix arguments
8414 inline bool Submatrix<MT,aligned,false,true,CSAs...>::isAligned() const noexcept
8415 {
8416  return true;
8417 }
8419 //*************************************************************************************************
8420 
8421 
8422 //*************************************************************************************************
8433 template< typename MT // Type of the dense matrix
8434  , size_t... CSAs > // Compile time submatrix arguments
8435 inline bool Submatrix<MT,aligned,false,true,CSAs...>::canSMPAssign() const noexcept
8436 {
8437  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
8438 }
8440 //*************************************************************************************************
8441 
8442 
8443 //*************************************************************************************************
8459 template< typename MT // Type of the dense matrix
8460  , size_t... CSAs > // Compile time submatrix arguments
8461 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,false,true,CSAs...>::SIMDType
8462  Submatrix<MT,aligned,false,true,CSAs...>::load( size_t i, size_t j ) const noexcept
8463 {
8464  return loada( i, j );
8465 }
8467 //*************************************************************************************************
8468 
8469 
8470 //*************************************************************************************************
8486 template< typename MT // Type of the dense matrix
8487  , size_t... CSAs > // Compile time submatrix arguments
8488 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,false,true,CSAs...>::SIMDType
8489  Submatrix<MT,aligned,false,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
8490 {
8492 
8493  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8494  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8495  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8496  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8497 
8498  return matrix_.loada( row()+i, column()+j );
8499 }
8501 //*************************************************************************************************
8502 
8503 
8504 //*************************************************************************************************
8520 template< typename MT // Type of the dense matrix
8521  , size_t... CSAs > // Compile time submatrix arguments
8522 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,false,true,CSAs...>::SIMDType
8523  Submatrix<MT,aligned,false,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
8524 {
8526 
8527  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8528  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8529  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8530  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8531 
8532  return matrix_.loadu( row()+i, column()+j );
8533 }
8535 //*************************************************************************************************
8536 
8537 
8538 //*************************************************************************************************
8555 template< typename MT // Type of the dense matrix
8556  , size_t... CSAs > // Compile time submatrix arguments
8558  Submatrix<MT,aligned,false,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
8559 {
8560  return storea( i, j, value );
8561 }
8563 //*************************************************************************************************
8564 
8565 
8566 //*************************************************************************************************
8583 template< typename MT // Type of the dense matrix
8584  , size_t... CSAs > // Compile time submatrix arguments
8586  Submatrix<MT,aligned,false,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
8587 {
8589 
8590  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8591  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8592  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8593  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8594 
8595  return matrix_.storea( row()+i, column()+j, value );
8596 }
8598 //*************************************************************************************************
8599 
8600 
8601 //*************************************************************************************************
8618 template< typename MT // Type of the dense matrix
8619  , size_t... CSAs > // Compile time submatrix arguments
8621  Submatrix<MT,aligned,false,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
8622 {
8624 
8625  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8626  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8627  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8628  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8629 
8630  matrix_.storeu( row()+i, column()+j, value );
8631 }
8633 //*************************************************************************************************
8634 
8635 
8636 //*************************************************************************************************
8654 template< typename MT // Type of the dense matrix
8655  , size_t... CSAs > // Compile time submatrix arguments
8657  Submatrix<MT,aligned,false,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
8658 {
8660 
8661  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
8662  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
8663  BLAZE_INTERNAL_ASSERT( j + SIMDSIZE <= columns(), "Invalid column access index" );
8664  BLAZE_INTERNAL_ASSERT( j % SIMDSIZE == 0UL, "Invalid column access index" );
8665 
8666  matrix_.stream( row()+i, column()+j, value );
8667 }
8669 //*************************************************************************************************
8670 
8671 
8672 //*************************************************************************************************
8684 template< typename MT // Type of the dense matrix
8685  , size_t... CSAs > // Compile time submatrix arguments
8686 template< typename MT2 > // Type of the right-hand side dense matrix
8687 inline auto Submatrix<MT,aligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
8688  -> DisableIf_t< VectorizedAssign_v<MT2> >
8689 {
8690  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8691  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8692 
8693  const size_t jpos( columns() & size_t(-2) );
8694  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
8695 
8696  for( size_t i=0UL; i<rows(); ++i ) {
8697  for( size_t j=0UL; j<jpos; j+=2UL ) {
8698  matrix_(row()+i,column()+j ) = (~rhs)(i,j );
8699  matrix_(row()+i,column()+j+1UL) = (~rhs)(i,j+1UL);
8700  }
8701  if( jpos < columns() ) {
8702  matrix_(row()+i,column()+jpos) = (~rhs)(i,jpos);
8703  }
8704  }
8705 }
8707 //*************************************************************************************************
8708 
8709 
8710 //*************************************************************************************************
8722 template< typename MT // Type of the dense matrix
8723  , size_t... CSAs > // Compile time submatrix arguments
8724 template< typename MT2 > // Type of the right-hand side dense matrix
8725 inline auto Submatrix<MT,aligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
8726  -> EnableIf_t< VectorizedAssign_v<MT2> >
8727 {
8729 
8730  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8731  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8732 
8733  const size_t jpos( columns() & size_t(-SIMDSIZE) );
8734  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
8735 
8736  if( useStreaming &&
8737  rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
8738  !(~rhs).isAliased( &matrix_ ) )
8739  {
8740  for( size_t i=0UL; i<rows(); ++i )
8741  {
8742  size_t j( 0UL );
8743  Iterator left( begin(i) );
8744  ConstIterator_t<MT2> right( (~rhs).begin(i) );
8745 
8746  for( ; j<jpos; j+=SIMDSIZE ) {
8747  left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8748  }
8749  for( ; j<columns(); ++j ) {
8750  *left = *right; ++left; ++right;
8751  }
8752  }
8753  }
8754  else
8755  {
8756  for( size_t i=0UL; i<rows(); ++i )
8757  {
8758  size_t j( 0UL );
8759  Iterator left( begin(i) );
8760  ConstIterator_t<MT2> right( (~rhs).begin(i) );
8761 
8762  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
8763  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8764  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8765  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8766  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8767  }
8768  for( ; j<jpos; j+=SIMDSIZE ) {
8769  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8770  }
8771  for( ; j<columns(); ++j ) {
8772  *left = *right; ++left; ++right;
8773  }
8774  }
8775  }
8776 }
8778 //*************************************************************************************************
8779 
8780 
8781 //*************************************************************************************************
8793 template< typename MT // Type of the dense matrix
8794  , size_t... CSAs > // Compile time submatrix arguments
8795 template< typename MT2 > // Type of the right-hand side dense matrix
8796 inline void Submatrix<MT,aligned,false,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
8797 {
8799 
8800  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8801  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8802 
8803  constexpr size_t block( BLOCK_SIZE );
8804 
8805  for( size_t ii=0UL; ii<rows(); ii+=block ) {
8806  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
8807  for( size_t jj=0UL; jj<columns(); jj+=block ) {
8808  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
8809  for( size_t i=ii; i<iend; ++i ) {
8810  for( size_t j=jj; j<jend; ++j ) {
8811  matrix_(row()+i,column()+j) = (~rhs)(i,j);
8812  }
8813  }
8814  }
8815  }
8816 }
8818 //*************************************************************************************************
8819 
8820 
8821 //*************************************************************************************************
8833 template< typename MT // Type of the dense matrix
8834  , size_t... CSAs > // Compile time submatrix arguments
8835 template< typename MT2 > // Type of the right-hand side sparse matrix
8836 inline void Submatrix<MT,aligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
8837 {
8838  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8839  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8840 
8841  for( size_t i=0UL; i<rows(); ++i )
8842  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
8843  matrix_(row()+i,column()+element->index()) = element->value();
8844 }
8846 //*************************************************************************************************
8847 
8848 
8849 //*************************************************************************************************
8861 template< typename MT // Type of the dense matrix
8862  , size_t... CSAs > // Compile time submatrix arguments
8863 template< typename MT2 > // Type of the right-hand side sparse matrix
8864 inline void Submatrix<MT,aligned,false,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
8865 {
8867 
8868  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8869  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8870 
8871  for( size_t j=0UL; j<columns(); ++j )
8872  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
8873  matrix_(row()+element->index(),column()+j) = element->value();
8874 }
8876 //*************************************************************************************************
8877 
8878 
8879 //*************************************************************************************************
8891 template< typename MT // Type of the dense matrix
8892  , size_t... CSAs > // Compile time submatrix arguments
8893 template< typename MT2 > // Type of the right-hand side dense matrix
8894 inline auto Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
8895  -> DisableIf_t< VectorizedAddAssign_v<MT2> >
8896 {
8897  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8898  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8899 
8900  const size_t jpos( columns() & size_t(-2) );
8901  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
8902 
8903  for( size_t i=0UL; i<rows(); ++i )
8904  {
8905  if( IsDiagonal_v<MT2> ) {
8906  matrix_(row()+i,column()+i) += (~rhs)(i,i);
8907  }
8908  else {
8909  for( size_t j=0UL; j<jpos; j+=2UL ) {
8910  matrix_(row()+i,column()+j ) += (~rhs)(i,j );
8911  matrix_(row()+i,column()+j+1UL) += (~rhs)(i,j+1UL);
8912  }
8913  if( jpos < columns() ) {
8914  matrix_(row()+i,column()+jpos) += (~rhs)(i,jpos);
8915  }
8916  }
8917  }
8918 }
8920 //*************************************************************************************************
8921 
8922 
8923 //*************************************************************************************************
8935 template< typename MT // Type of the dense matrix
8936  , size_t... CSAs > // Compile time submatrix arguments
8937 template< typename MT2 > // Type of the right-hand side dense matrix
8938 inline auto Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
8939  -> EnableIf_t< VectorizedAddAssign_v<MT2> >
8940 {
8942 
8943  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
8944  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
8945 
8946  for( size_t i=0UL; i<rows(); ++i )
8947  {
8948  const size_t jbegin( ( IsUpper_v<MT2> )
8949  ?( ( IsStrictlyUpper_v<MT2> ? i+1UL : i ) & size_t(-SIMDSIZE) )
8950  :( 0UL ) );
8951  const size_t jend ( ( IsLower_v<MT2> )
8952  ?( IsStrictlyLower_v<MT2> ? i : i+1UL )
8953  :( columns() ) );
8954  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
8955 
8956  const size_t jpos( jend & size_t(-SIMDSIZE) );
8957  BLAZE_INTERNAL_ASSERT( ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
8958 
8959  size_t j( jbegin );
8960  Iterator left( begin(i) + jbegin );
8961  ConstIterator_t<MT2> right( (~rhs).begin(i) + jbegin );
8962 
8963  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
8964  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8965  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8966  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8967  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8968  }
8969  for( ; j<jpos; j+=SIMDSIZE ) {
8970  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
8971  }
8972  for( ; j<jend; ++j ) {
8973  *left += *right; ++left; ++right;
8974  }
8975  }
8976 }
8978 //*************************************************************************************************
8979 
8980 
8981 //*************************************************************************************************
8993 template< typename MT // Type of the dense matrix
8994  , size_t... CSAs > // Compile time submatrix arguments
8995 template< typename MT2 > // Type of the right-hand side dense matrix
8996 inline void Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
8997 {
8999 
9000  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9001  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9002 
9003  constexpr size_t block( BLOCK_SIZE );
9004 
9005  for( size_t ii=0UL; ii<rows(); ii+=block ) {
9006  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
9007  for( size_t jj=0UL; jj<columns(); jj+=block ) {
9008  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
9009  for( size_t i=ii; i<iend; ++i ) {
9010  for( size_t j=jj; j<jend; ++j ) {
9011  matrix_(row()+i,column()+j) += (~rhs)(i,j);
9012  }
9013  }
9014  }
9015  }
9016 }
9018 //*************************************************************************************************
9019 
9020 
9021 //*************************************************************************************************
9033 template< typename MT // Type of the dense matrix
9034  , size_t... CSAs > // Compile time submatrix arguments
9035 template< typename MT2 > // Type of the right-hand side sparse matrix
9036 inline void Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
9037 {
9038  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9039  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9040 
9041  for( size_t i=0UL; i<rows(); ++i )
9042  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
9043  matrix_(row()+i,column()+element->index()) += element->value();
9044 }
9046 //*************************************************************************************************
9047 
9048 
9049 //*************************************************************************************************
9061 template< typename MT // Type of the dense matrix
9062  , size_t... CSAs > // Compile time submatrix arguments
9063 template< typename MT2 > // Type of the right-hand side sparse matrix
9064 inline void Submatrix<MT,aligned,false,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
9065 {
9067 
9068  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9069  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9070 
9071  for( size_t j=0UL; j<columns(); ++j )
9072  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
9073  matrix_(row()+element->index(),column()+j) += element->value();
9074 }
9076 //*************************************************************************************************
9077 
9078 
9079 //*************************************************************************************************
9091 template< typename MT // Type of the dense matrix
9092  , size_t... CSAs > // Compile time submatrix arguments
9093 template< typename MT2 > // Type of the right-hand side dense matrix
9094 inline auto Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
9095  -> DisableIf_t< VectorizedSubAssign_v<MT2> >
9096 {
9097  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9098  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9099 
9100  const size_t jpos( columns() & size_t(-2) );
9101  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
9102 
9103  for( size_t i=0UL; i<rows(); ++i )
9104  {
9105  if( IsDiagonal_v<MT2> ) {
9106  matrix_(row()+i,column()+i) -= (~rhs)(i,i);
9107  }
9108  else {
9109  for( size_t j=0UL; j<jpos; j+=2UL ) {
9110  matrix_(row()+i,column()+j ) -= (~rhs)(i,j );
9111  matrix_(row()+i,column()+j+1UL) -= (~rhs)(i,j+1UL);
9112  }
9113  if( jpos < columns() ) {
9114  matrix_(row()+i,column()+jpos) -= (~rhs)(i,jpos);
9115  }
9116  }
9117  }
9118 }
9120 //*************************************************************************************************
9121 
9122 
9123 //*************************************************************************************************
9135 template< typename MT // Type of the dense matrix
9136  , size_t... CSAs > // Compile time submatrix arguments
9137 template< typename MT2 > // Type of the right-hand side dense matrix
9138 inline auto Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
9139  -> EnableIf_t< VectorizedSubAssign_v<MT2> >
9140 {
9142 
9143  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9144  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9145 
9146  for( size_t i=0UL; i<rows(); ++i )
9147  {
9148  const size_t jbegin( ( IsUpper_v<MT2> )
9149  ?( ( IsStrictlyUpper_v<MT2> ? i+1UL : i ) & size_t(-SIMDSIZE) )
9150  :( 0UL ) );
9151  const size_t jend ( ( IsLower_v<MT2> )
9152  ?( IsStrictlyLower_v<MT2> ? i : i+1UL )
9153  :( columns() ) );
9154  BLAZE_INTERNAL_ASSERT( jbegin <= jend, "Invalid loop indices detected" );
9155 
9156  const size_t jpos( jend & size_t(-SIMDSIZE) );
9157  BLAZE_INTERNAL_ASSERT( ( jend - ( jend % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
9158 
9159  size_t j( jbegin );
9160  Iterator left( begin(i) + jbegin );
9161  ConstIterator_t<MT2> right( (~rhs).begin(i) + jbegin );
9162 
9163  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
9164  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9165  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9166  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9167  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9168  }
9169  for( ; j<jpos; j+=SIMDSIZE ) {
9170  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9171  }
9172  for( ; j<jend; ++j ) {
9173  *left -= *right; ++left; ++right;
9174  }
9175  }
9176 }
9178 //*************************************************************************************************
9179 
9180 
9181 //*************************************************************************************************
9193 template< typename MT // Type of the dense matrix
9194  , size_t... CSAs > // Compile time submatrix arguments
9195 template< typename MT2 > // Type of the right-hand side dense matrix
9196 inline void Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
9197 {
9199 
9200  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9201  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9202 
9203  constexpr size_t block( BLOCK_SIZE );
9204 
9205  for( size_t ii=0UL; ii<rows(); ii+=block ) {
9206  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
9207  for( size_t jj=0UL; jj<columns(); jj+=block ) {
9208  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
9209  for( size_t i=ii; i<iend; ++i ) {
9210  for( size_t j=jj; j<jend; ++j ) {
9211  matrix_(row()+i,column()+j) -= (~rhs)(i,j);
9212  }
9213  }
9214  }
9215  }
9216 }
9218 //*************************************************************************************************
9219 
9220 
9221 //*************************************************************************************************
9233 template< typename MT // Type of the dense matrix
9234  , size_t... CSAs > // Compile time submatrix arguments
9235 template< typename MT2 > // Type of the right-hand side sparse matrix
9236 inline void Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
9237 {
9238  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9239  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9240 
9241  for( size_t i=0UL; i<rows(); ++i )
9242  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
9243  matrix_(row()+i,column()+element->index()) -= element->value();
9244 }
9246 //*************************************************************************************************
9247 
9248 
9249 //*************************************************************************************************
9261 template< typename MT // Type of the dense matrix
9262  , size_t... CSAs > // Compile time submatrix arguments
9263 template< typename MT2 > // Type of the right-hand side sparse matrix
9264 inline void Submatrix<MT,aligned,false,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
9265 {
9267 
9268  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9269  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9270 
9271  for( size_t j=0UL; j<columns(); ++j )
9272  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
9273  matrix_(row()+element->index(),column()+j) -= element->value();
9274 }
9276 //*************************************************************************************************
9277 
9278 
9279 //*************************************************************************************************
9291 template< typename MT // Type of the dense matrix
9292  , size_t... CSAs > // Compile time submatrix arguments
9293 template< typename MT2 > // Type of the right-hand side dense matrix
9294 inline auto Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
9295  -> DisableIf_t< VectorizedSchurAssign_v<MT2> >
9296 {
9297  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9298  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9299 
9300  const size_t jpos( columns() & size_t(-2) );
9301  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % 2UL ) ) == jpos, "Invalid end calculation" );
9302 
9303  for( size_t i=0UL; i<rows(); ++i ) {
9304  for( size_t j=0UL; j<jpos; j+=2UL ) {
9305  matrix_(row()+i,column()+j ) *= (~rhs)(i,j );
9306  matrix_(row()+i,column()+j+1UL) *= (~rhs)(i,j+1UL);
9307  }
9308  if( jpos < columns() ) {
9309  matrix_(row()+i,column()+jpos) *= (~rhs)(i,jpos);
9310  }
9311  }
9312 }
9314 //*************************************************************************************************
9315 
9316 
9317 //*************************************************************************************************
9329 template< typename MT // Type of the dense matrix
9330  , size_t... CSAs > // Compile time submatrix arguments
9331 template< typename MT2 > // Type of the right-hand side dense matrix
9332 inline auto Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
9333  -> EnableIf_t< VectorizedSchurAssign_v<MT2> >
9334 {
9336 
9337  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9338  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9339 
9340  for( size_t i=0UL; i<rows(); ++i )
9341  {
9342  const size_t jpos( columns() & size_t(-SIMDSIZE) );
9343  BLAZE_INTERNAL_ASSERT( ( columns() - ( columns() % (SIMDSIZE) ) ) == jpos, "Invalid end calculation" );
9344 
9345  size_t j( 0UL );
9346  Iterator left( begin(i) );
9347  ConstIterator_t<MT2> right( (~rhs).begin(i) );
9348 
9349  for( ; (j+SIMDSIZE*3UL) < jpos; j+=SIMDSIZE*4UL ) {
9350  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9351  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9352  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9353  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9354  }
9355  for( ; j<jpos; j+=SIMDSIZE ) {
9356  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
9357  }
9358  for( ; j<columns(); ++j ) {
9359  *left *= *right; ++left; ++right;
9360  }
9361  }
9362 }
9364 //*************************************************************************************************
9365 
9366 
9367 //*************************************************************************************************
9379 template< typename MT // Type of the dense matrix
9380  , size_t... CSAs > // Compile time submatrix arguments
9381 template< typename MT2 > // Type of the right-hand side dense matrix
9382 inline void Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
9383 {
9385 
9386  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9387  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9388 
9389  constexpr size_t block( BLOCK_SIZE );
9390 
9391  for( size_t ii=0UL; ii<rows(); ii+=block ) {
9392  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
9393  for( size_t jj=0UL; jj<columns(); jj+=block ) {
9394  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
9395  for( size_t i=ii; i<iend; ++i ) {
9396  for( size_t j=jj; j<jend; ++j ) {
9397  matrix_(row()+i,column()+j) *= (~rhs)(i,j);
9398  }
9399  }
9400  }
9401  }
9402 }
9404 //*************************************************************************************************
9405 
9406 
9407 //*************************************************************************************************
9419 template< typename MT // Type of the dense matrix
9420  , size_t... CSAs > // Compile time submatrix arguments
9421 template< typename MT2 > // Type of the right-hand side sparse matrix
9422 inline void Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
9423 {
9424  using blaze::reset;
9425 
9426  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9427  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9428 
9429  for( size_t i=0UL; i<rows(); ++i )
9430  {
9431  size_t j( 0UL );
9432 
9433  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element ) {
9434  for( ; j<element->index(); ++j )
9435  reset( matrix_(row()+i,column()+j) );
9436  matrix_(row()+i,column()+j) *= element->value();
9437  ++j;
9438  }
9439 
9440  for( ; j<columns(); ++j ) {
9441  reset( matrix_(row()+i,column()+j) );
9442  }
9443  }
9444 }
9446 //*************************************************************************************************
9447 
9448 
9449 //*************************************************************************************************
9461 template< typename MT // Type of the dense matrix
9462  , size_t... CSAs > // Compile time submatrix arguments
9463 template< typename MT2 > // Type of the right-hand side sparse matrix
9464 inline void Submatrix<MT,aligned,false,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
9465 {
9466  using blaze::reset;
9467 
9469 
9470  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
9471  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
9472 
9473  for( size_t j=0UL; j<columns(); ++j )
9474  {
9475  size_t i( 0UL );
9476 
9477  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element ) {
9478  for( ; i<element->index(); ++i )
9479  reset( matrix_(row()+i,column()+j) );
9480  matrix_(row()+element->index(),column()+j) *= element->value();
9481  ++i;
9482  }
9483 
9484  for( ; i<rows(); ++i ) {
9485  reset( matrix_(row()+i,column()+j) );
9486  }
9487  }
9488 }
9490 //*************************************************************************************************
9491 
9492 
9493 
9494 
9495 
9496 
9497 
9498 
9499 //=================================================================================================
9500 //
9501 // CLASS TEMPLATE SPECIALIZATION FOR ALIGNED COLUMN-MAJOR DENSE SUBMATRICES
9502 //
9503 //=================================================================================================
9504 
9505 //*************************************************************************************************
9513 template< typename MT // Type of the dense matrix
9514  , size_t... CSAs > // Compile time submatrix arguments
9515 class Submatrix<MT,aligned,true,true,CSAs...>
9516  : public View< DenseMatrix< Submatrix<MT,aligned,true,true,CSAs...>, true > >
9517  , private SubmatrixData<CSAs...>
9518 {
9519  private:
9520  //**Type definitions****************************************************************************
9521  using DataType = SubmatrixData<CSAs...>;
9522  using Operand = If_t< IsExpression_v<MT>, MT, MT& >;
9523  //**********************************************************************************************
9524 
9525  //**********************************************************************************************
9527  template< typename MT1, typename MT2 >
9528  static constexpr bool EnforceEvaluation_v =
9529  ( IsRestricted_v<MT1> && RequiresEvaluation_v<MT2> );
9530  //**********************************************************************************************
9531 
9532  public:
9533  //**Type definitions****************************************************************************
9535  using This = Submatrix<MT,aligned,true,true,CSAs...>;
9536 
9537  using BaseType = DenseMatrix<This,true>;
9538  using ViewedType = MT;
9539  using ResultType = SubmatrixTrait_t<MT,CSAs...>;
9540  using OppositeType = OppositeType_t<ResultType>;
9541  using TransposeType = TransposeType_t<ResultType>;
9542  using ElementType = ElementType_t<MT>;
9543  using SIMDType = SIMDTrait_t<ElementType>;
9544  using ReturnType = ReturnType_t<MT>;
9545  using CompositeType = const Submatrix&;
9546 
9548  using ConstReference = ConstReference_t<MT>;
9549 
9551  using Reference = If_t< IsConst_v<MT>, ConstReference, Reference_t<MT> >;
9552 
9554  using ConstPointer = ConstPointer_t<MT>;
9555 
9557  using Pointer = If_t< IsConst_v<MT> || !HasMutableDataAccess_v<MT>, ConstPointer, Pointer_t<MT> >;
9558 
9560  using ConstIterator = ConstIterator_t<MT>;
9561 
9563  using Iterator = If_t< IsConst_v<MT>, ConstIterator, Iterator_t<MT> >;
9564  //**********************************************************************************************
9565 
9566  //**Compilation flags***************************************************************************
9568  static constexpr bool simdEnabled = MT::simdEnabled;
9569 
9571  static constexpr bool smpAssignable = MT::smpAssignable;
9572  //**********************************************************************************************
9573 
9574  //**Constructors********************************************************************************
9577  template< typename... RSAs >
9578  explicit inline Submatrix( MT& matrix, RSAs... args );
9579 
9580  Submatrix( const Submatrix& ) = default;
9582  //**********************************************************************************************
9583 
9584  //**Destructor**********************************************************************************
9587  ~Submatrix() = default;
9589  //**********************************************************************************************
9590 
9591  //**Data access functions***********************************************************************
9594  inline Reference operator()( size_t i, size_t j );
9595  inline ConstReference operator()( size_t i, size_t j ) const;
9596  inline Reference at( size_t i, size_t j );
9597  inline ConstReference at( size_t i, size_t j ) const;
9598  inline Pointer data () noexcept;
9599  inline ConstPointer data () const noexcept;
9600  inline Pointer data ( size_t j ) noexcept;
9601  inline ConstPointer data ( size_t j ) const noexcept;
9602  inline Iterator begin ( size_t j );
9603  inline ConstIterator begin ( size_t j ) const;
9604  inline ConstIterator cbegin( size_t j ) const;
9605  inline Iterator end ( size_t j );
9606  inline ConstIterator end ( size_t j ) const;
9607  inline ConstIterator cend ( size_t j ) const;
9609  //**********************************************************************************************
9610 
9611  //**Assignment operators************************************************************************
9614  inline Submatrix& operator=( const ElementType& rhs );
9615  inline Submatrix& operator=( initializer_list< initializer_list<ElementType> > list );
9616  inline Submatrix& operator=( const Submatrix& rhs );
9617 
9618  template< typename MT2, bool SO >
9619  inline Submatrix& operator=( const Matrix<MT2,SO>& rhs );
9620 
9621  template< typename MT2, bool SO >
9622  inline auto operator+=( const Matrix<MT2,SO>& rhs )
9623  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9624 
9625  template< typename MT2, bool SO >
9626  inline auto operator+=( const Matrix<MT2,SO>& rhs )
9627  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9628 
9629  template< typename MT2, bool SO >
9630  inline auto operator-=( const Matrix<MT2,SO>& rhs )
9631  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9632 
9633  template< typename MT2, bool SO >
9634  inline auto operator-=( const Matrix<MT2,SO>& rhs )
9635  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9636 
9637  template< typename MT2, bool SO >
9638  inline auto operator%=( const Matrix<MT2,SO>& rhs )
9639  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9640 
9641  template< typename MT2, bool SO >
9642  inline auto operator%=( const Matrix<MT2,SO>& rhs )
9643  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >;
9645  //**********************************************************************************************
9646 
9647  //**Utility functions***************************************************************************
9650  using DataType::row;
9651  using DataType::column;
9652  using DataType::rows;
9653  using DataType::columns;
9654 
9655  inline MT& operand() noexcept;
9656  inline const MT& operand() const noexcept;
9657 
9658  inline size_t spacing() const noexcept;
9659  inline size_t capacity() const noexcept;
9660  inline size_t capacity( size_t i ) const noexcept;
9661  inline size_t nonZeros() const;
9662  inline size_t nonZeros( size_t i ) const;
9663  inline void reset();
9664  inline void reset( size_t i );
9666  //**********************************************************************************************
9667 
9668  //**Numeric functions***************************************************************************
9671  inline Submatrix& transpose();
9672  inline Submatrix& ctranspose();
9673 
9674  template< typename Other > inline Submatrix& scale( const Other& scalar );
9676  //**********************************************************************************************
9677 
9678  private:
9679  //**********************************************************************************************
9681  template< typename MT2 >
9682  static constexpr bool VectorizedAssign_v =
9683  ( useOptimizedKernels &&
9684  simdEnabled && MT2::simdEnabled &&
9686  //**********************************************************************************************
9687 
9688  //**********************************************************************************************
9690  template< typename MT2 >
9691  static constexpr bool VectorizedAddAssign_v =
9692  ( useOptimizedKernels &&
9693  simdEnabled && MT2::simdEnabled &&
9696  !IsDiagonal_v<MT2> );
9697  //**********************************************************************************************
9698 
9699  //**********************************************************************************************
9701  template< typename MT2 >
9702  static constexpr bool VectorizedSubAssign_v =
9703  ( useOptimizedKernels &&
9704  simdEnabled && MT2::simdEnabled &&
9707  !IsDiagonal_v<MT2> );
9708  //**********************************************************************************************
9709 
9710  //**********************************************************************************************
9712  template< typename MT2 >
9713  static constexpr bool VectorizedSchurAssign_v =
9714  ( useOptimizedKernels &&
9715  simdEnabled && MT2::simdEnabled &&
9718  //**********************************************************************************************
9719 
9720  //**SIMD properties*****************************************************************************
9722  static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
9723  //**********************************************************************************************
9724 
9725  public:
9726  //**Expression template evaluation functions****************************************************
9729  template< typename Other >
9730  inline bool canAlias( const Other* alias ) const noexcept;
9731 
9732  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
9733  inline bool canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
9734 
9735  template< typename Other >
9736  inline bool isAliased( const Other* alias ) const noexcept;
9737 
9738  template< typename MT2, AlignmentFlag AF2, bool SO2, size_t... CSAs2 >
9739  inline bool isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept;
9740 
9741  inline bool isAligned () const noexcept;
9742  inline bool canSMPAssign() const noexcept;
9743 
9744  BLAZE_ALWAYS_INLINE SIMDType load ( size_t i, size_t j ) const noexcept;
9745  BLAZE_ALWAYS_INLINE SIMDType loada( size_t i, size_t j ) const noexcept;
9746  BLAZE_ALWAYS_INLINE SIMDType loadu( size_t i, size_t j ) const noexcept;
9747 
9748  BLAZE_ALWAYS_INLINE void store ( size_t i, size_t j, const SIMDType& value ) noexcept;
9749  BLAZE_ALWAYS_INLINE void storea( size_t i, size_t j, const SIMDType& value ) noexcept;
9750  BLAZE_ALWAYS_INLINE void storeu( size_t i, size_t j, const SIMDType& value ) noexcept;
9751  BLAZE_ALWAYS_INLINE void stream( size_t i, size_t j, const SIMDType& value ) noexcept;
9752 
9753  template< typename MT2 >
9754  inline auto assign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedAssign_v<MT2> >;
9755 
9756  template< typename MT2 >
9757  inline auto assign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedAssign_v<MT2> >;
9758 
9759  template< typename MT2 > inline void assign( const DenseMatrix<MT2,false>& rhs );
9760  template< typename MT2 > inline void assign( const SparseMatrix<MT2,true>& rhs );
9761  template< typename MT2 > inline void assign( const SparseMatrix<MT2,false>& rhs );
9762 
9763  template< typename MT2 >
9764  inline auto addAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedAddAssign_v<MT2> >;
9765 
9766  template< typename MT2 >
9767  inline auto addAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedAddAssign_v<MT2> >;
9768 
9769  template< typename MT2 > inline void addAssign( const DenseMatrix<MT2,false>& rhs );
9770  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,true>& rhs );
9771  template< typename MT2 > inline void addAssign( const SparseMatrix<MT2,false>& rhs );
9772 
9773  template< typename MT2 >
9774  inline auto subAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedSubAssign_v<MT2> >;
9775 
9776  template< typename MT2 >
9777  inline auto subAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedSubAssign_v<MT2> >;
9778 
9779  template< typename MT2 > inline void subAssign( const DenseMatrix<MT2,false>& rhs );
9780  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,true>& rhs );
9781  template< typename MT2 > inline void subAssign( const SparseMatrix<MT2,false>& rhs );
9782 
9783  template< typename MT2 >
9784  inline auto schurAssign( const DenseMatrix<MT2,true>& rhs ) -> DisableIf_t< VectorizedSchurAssign_v<MT2> >;
9785 
9786  template< typename MT2 >
9787  inline auto schurAssign( const DenseMatrix<MT2,true>& rhs ) -> EnableIf_t< VectorizedSchurAssign_v<MT2> >;
9788 
9789  template< typename MT2 > inline void schurAssign( const DenseMatrix<MT2,false>& rhs );
9790  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,true>& rhs );
9791  template< typename MT2 > inline void schurAssign( const SparseMatrix<MT2,false>& rhs );
9793  //**********************************************************************************************
9794 
9795  private:
9796  //**Utility functions***************************************************************************
9799  inline bool hasOverlap() const noexcept;
9801  //**********************************************************************************************
9802 
9803  //**Member variables****************************************************************************
9806  Operand matrix_;
9807 
9808  //**********************************************************************************************
9809 
9810  //**Friend declarations*************************************************************************
9811  template< typename MT2, AlignmentFlag AF2, bool SO2, bool DF2, size_t... CSAs2 > friend class Submatrix;
9812  //**********************************************************************************************
9813 
9814  //**Compile time checks*************************************************************************
9822  //**********************************************************************************************
9823 };
9825 //*************************************************************************************************
9826 
9827 
9828 
9829 
9830 //=================================================================================================
9831 //
9832 // CONSTRUCTORS
9833 //
9834 //=================================================================================================
9835 
9836 //*************************************************************************************************
9849 template< typename MT // Type of the dense matrix
9850  , size_t... CSAs > // Compile time submatrix arguments
9851 template< typename... RSAs > // Runtime submatrix arguments
9852 inline Submatrix<MT,aligned,true,true,CSAs...>::Submatrix( MT& matrix, RSAs... args )
9853  : DataType( args... ) // Base class initialization
9854  , matrix_ ( matrix ) // The matrix containing the submatrix
9855 {
9856  if( !Contains_v< TypeList<RSAs...>, Unchecked > )
9857  {
9858  if( ( row() + rows() > matrix_.rows() ) || ( column() + columns() > matrix_.columns() ) ) {
9859  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix specification" );
9860  }
9861 
9862  if( simdEnabled && IsContiguous_v<MT> &&
9863  ( !checkAlignment( data() ) ||
9864  ( columns() > 1UL && matrix_.spacing() % SIMDSIZE != 0UL ) ) ) {
9865  BLAZE_THROW_INVALID_ARGUMENT( "Invalid submatrix alignment" );
9866  }
9867  }
9868  else
9869  {
9870  BLAZE_USER_ASSERT( row() + rows() <= matrix_.rows() , "Invalid submatrix specification" );
9871  BLAZE_USER_ASSERT( column() + columns() <= matrix_.columns(), "Invalid submatrix specification" );
9872 
9873  BLAZE_USER_ASSERT( !simdEnabled || !IsContiguous_v<MT> || checkAlignment( data() ), "Invalid submatrix alignment" );
9874  BLAZE_USER_ASSERT( !simdEnabled || !IsContiguous_v<MT> || columns() <= 1UL || matrix_.spacing() % SIMDSIZE == 0UL, "Invalid submatrix alignment" );
9875  }
9876 }
9878 //*************************************************************************************************
9879 
9880 
9881 
9882 
9883 //=================================================================================================
9884 //
9885 // DATA ACCESS FUNCTIONS
9886 //
9887 //=================================================================================================
9888 
9889 //*************************************************************************************************
9900 template< typename MT // Type of the dense matrix
9901  , size_t... CSAs > // Compile time submatrix arguments
9902 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Reference
9903  Submatrix<MT,aligned,true,true,CSAs...>::operator()( size_t i, size_t j )
9904 {
9905  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
9906  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
9907 
9908  return matrix_(row()+i,column()+j);
9909 }
9911 //*************************************************************************************************
9912 
9913 
9914 //*************************************************************************************************
9925 template< typename MT // Type of the dense matrix
9926  , size_t... CSAs > // Compile time submatrix arguments
9927 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstReference
9928  Submatrix<MT,aligned,true,true,CSAs...>::operator()( size_t i, size_t j ) const
9929 {
9930  BLAZE_USER_ASSERT( i < rows() , "Invalid row access index" );
9931  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
9932 
9933  return const_cast<const MT&>( matrix_ )(row()+i,column()+j);
9934 }
9936 //*************************************************************************************************
9937 
9938 
9939 //*************************************************************************************************
9951 template< typename MT // Type of the dense matrix
9952  , size_t... CSAs > // Compile time submatrix arguments
9953 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Reference
9954  Submatrix<MT,aligned,true,true,CSAs...>::at( size_t i, size_t j )
9955 {
9956  if( i >= rows() ) {
9957  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
9958  }
9959  if( j >= columns() ) {
9960  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
9961  }
9962  return (*this)(i,j);
9963 }
9965 //*************************************************************************************************
9966 
9967 
9968 //*************************************************************************************************
9980 template< typename MT // Type of the dense matrix
9981  , size_t... CSAs > // Compile time submatrix arguments
9982 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstReference
9983  Submatrix<MT,aligned,true,true,CSAs...>::at( size_t i, size_t j ) const
9984 {
9985  if( i >= rows() ) {
9986  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
9987  }
9988  if( j >= columns() ) {
9989  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
9990  }
9991  return (*this)(i,j);
9992 }
9994 //*************************************************************************************************
9995 
9996 
9997 //*************************************************************************************************
10007 template< typename MT // Type of the dense matrix
10008  , size_t... CSAs > // Compile time submatrix arguments
10009 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Pointer
10011 {
10012  return matrix_.data() + row() + column()*spacing();
10013 }
10015 //*************************************************************************************************
10016 
10017 
10018 //*************************************************************************************************
10028 template< typename MT // Type of the dense matrix
10029  , size_t... CSAs > // Compile time submatrix arguments
10030 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstPointer
10032 {
10033  return matrix_.data() + row() + column()*spacing();
10034 }
10036 //*************************************************************************************************
10037 
10038 
10039 //*************************************************************************************************
10048 template< typename MT // Type of the dense matrix
10049  , size_t... CSAs > // Compile time submatrix arguments
10050 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Pointer
10052 {
10053  return matrix_.data() + row() + (column()+j)*spacing();
10054 }
10056 //*************************************************************************************************
10057 
10058 
10059 //*************************************************************************************************
10068 template< typename MT // Type of the dense matrix
10069  , size_t... CSAs > // Compile time submatrix arguments
10070 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstPointer
10071  Submatrix<MT,aligned,true,true,CSAs...>::data( size_t j ) const noexcept
10072 {
10073  return matrix_.data() + row() + (column()+j)*spacing();
10074 }
10076 //*************************************************************************************************
10077 
10078 
10079 //*************************************************************************************************
10086 template< typename MT // Type of the dense matrix
10087  , size_t... CSAs > // Compile time submatrix arguments
10088 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Iterator
10090 {
10091  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10092  return ( matrix_.begin( column() + j ) + row() );
10093 }
10095 //*************************************************************************************************
10096 
10097 
10098 //*************************************************************************************************
10105 template< typename MT // Type of the dense matrix
10106  , size_t... CSAs > // Compile time submatrix arguments
10107 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10109 {
10110  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10111  return ( matrix_.cbegin( column() + j ) + row() );
10112 }
10114 //*************************************************************************************************
10115 
10116 
10117 //*************************************************************************************************
10124 template< typename MT // Type of the dense matrix
10125  , size_t... CSAs > // Compile time submatrix arguments
10126 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10128 {
10129  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10130  return ( matrix_.cbegin( column() + j ) + row() );
10131 }
10133 //*************************************************************************************************
10134 
10135 
10136 //*************************************************************************************************
10143 template< typename MT // Type of the dense matrix
10144  , size_t... CSAs > // Compile time submatrix arguments
10145 inline typename Submatrix<MT,aligned,true,true,CSAs...>::Iterator
10147 {
10148  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10149  return ( matrix_.begin( column() + j ) + row() + rows() );
10150 }
10152 //*************************************************************************************************
10153 
10154 
10155 //*************************************************************************************************
10162 template< typename MT // Type of the dense matrix
10163  , size_t... CSAs > // Compile time submatrix arguments
10164 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10166 {
10167  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10168  return ( matrix_.cbegin( column() + j ) + row() + rows() );
10169 }
10171 //*************************************************************************************************
10172 
10173 
10174 //*************************************************************************************************
10181 template< typename MT // Type of the dense matrix
10182  , size_t... CSAs > // Compile time submatrix arguments
10183 inline typename Submatrix<MT,aligned,true,true,CSAs...>::ConstIterator
10185 {
10186  BLAZE_USER_ASSERT( j < columns(), "Invalid dense submatrix column access index" );
10187  return ( matrix_.cbegin( column() + j ) + row() + rows() );
10188 }
10190 //*************************************************************************************************
10191 
10192 
10193 
10194 
10195 //=================================================================================================
10196 //
10197 // ASSIGNMENT OPERATORS
10198 //
10199 //=================================================================================================
10200 
10201 //*************************************************************************************************
10212 template< typename MT // Type of the dense matrix
10213  , size_t... CSAs > // Compile time submatrix arguments
10214 inline Submatrix<MT,aligned,true,true,CSAs...>&
10215  Submatrix<MT,aligned,true,true,CSAs...>::operator=( const ElementType& rhs )
10216 {
10217  const size_t jend( column() + columns() );
10218  decltype(auto) left( derestrict( matrix_ ) );
10219 
10220  for( size_t j=column(); j<jend; ++j )
10221  {
10222  const size_t ibegin( ( IsLower_v<MT> )
10223  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
10224  ?( max( j+1UL, row() ) )
10225  :( max( j, row() ) ) )
10226  :( row() ) );
10227  const size_t iend ( ( IsUpper_v<MT> )
10228  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
10229  ?( min( j, row()+rows() ) )
10230  :( min( j+1UL, row()+rows() ) ) )
10231  :( row()+rows() ) );
10232 
10233  for( size_t i=ibegin; i<iend; ++i ) {
10234  if( !IsRestricted_v<MT> || IsTriangular_v<MT> || trySet( matrix_, i, j, rhs ) )
10235  left(i,j) = rhs;
10236  }
10237  }
10238 
10239  return *this;
10240 }
10242 //*************************************************************************************************
10243 
10244 
10245 //*************************************************************************************************
10261 template< typename MT // Type of the dense matrix
10262  , size_t... CSAs > // Compile time submatrix arguments
10263 inline Submatrix<MT,aligned,true,true,CSAs...>&
10264  Submatrix<MT,aligned,true,true,CSAs...>::operator=( initializer_list< initializer_list<ElementType> > list )
10265 {
10266  using blaze::reset;
10267 
10268  if( list.size() != rows() ) {
10269  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to submatrix" );
10270  }
10271 
10272  if( IsRestricted_v<MT> ) {
10273  const InitializerMatrix<ElementType> tmp( list, columns() );
10274  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10275  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10276  }
10277  }
10278 
10279  decltype(auto) left( derestrict( *this ) );
10280  size_t i( 0UL );
10281 
10282  for( const auto& rowList : list ) {
10283  size_t j( 0UL );
10284  for( const auto& element : rowList ) {
10285  left(i,j) = element;
10286  ++j;
10287  }
10288  for( ; j<columns(); ++j ) {
10289  reset( left(i,j) );
10290  }
10291  ++i;
10292  }
10293 
10294  return *this;
10295 }
10297 //*************************************************************************************************
10298 
10299 
10300 //*************************************************************************************************
10315 template< typename MT // Type of the dense matrix
10316  , size_t... CSAs > // Compile time submatrix arguments
10317 inline Submatrix<MT,aligned,true,true,CSAs...>&
10318  Submatrix<MT,aligned,true,true,CSAs...>::operator=( const Submatrix& rhs )
10319 {
10322 
10323  if( this == &rhs || ( &matrix_ == &rhs.matrix_ && row() == rhs.row() && column() == rhs.column() ) )
10324  return *this;
10325 
10326  if( rows() != rhs.rows() || columns() != rhs.columns() ) {
10327  BLAZE_THROW_INVALID_ARGUMENT( "Submatrix sizes do not match" );
10328  }
10329 
10330  if( !tryAssign( matrix_, rhs, row(), column() ) ) {
10331  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10332  }
10333 
10334  decltype(auto) left( derestrict( *this ) );
10335 
10336  if( rhs.canAlias( &matrix_ ) ) {
10337  const ResultType tmp( rhs );
10338  smpAssign( left, tmp );
10339  }
10340  else {
10341  smpAssign( left, rhs );
10342  }
10343 
10344  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10345 
10346  return *this;
10347 }
10349 //*************************************************************************************************
10350 
10351 
10352 //*************************************************************************************************
10366 template< typename MT // Type of the dense matrix
10367  , size_t... CSAs > // Compile time submatrix arguments
10368 template< typename MT2 // Type of the right-hand side matrix
10369  , bool SO > // Storage order of the right-hand side matrix
10370 inline Submatrix<MT,aligned,true,true,CSAs...>&
10371  Submatrix<MT,aligned,true,true,CSAs...>::operator=( const Matrix<MT2,SO>& rhs )
10372 {
10373  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10374 
10375  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10376  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10377  }
10378 
10379  using Right = If_t< IsRestricted_v<MT>, CompositeType_t<MT2>, const MT2& >;
10380  Right right( ~rhs );
10381 
10382  if( !tryAssign( matrix_, right, row(), column() ) ) {
10383  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10384  }
10385 
10386  decltype(auto) left( derestrict( *this ) );
10387 
10388  if( IsReference_v<Right> && right.canAlias( &matrix_ ) ) {
10389  const ResultType_t<MT2> tmp( right );
10390  if( IsSparseMatrix_v<MT2> )
10391  reset();
10392  smpAssign( left, tmp );
10393  }
10394  else {
10395  if( IsSparseMatrix_v<MT2> )
10396  reset();
10397  smpAssign( left, right );
10398  }
10399 
10400  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10401 
10402  return *this;
10403 }
10405 //*************************************************************************************************
10406 
10407 
10408 //*************************************************************************************************
10422 template< typename MT // Type of the dense matrix
10423  , size_t... CSAs > // Compile time submatrix arguments
10424 template< typename MT2 // Type of the right-hand side matrix
10425  , bool SO > // Storage order of the right-hand side matrix
10426 inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
10427  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10428 {
10431  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10432 
10433  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
10434 
10437 
10438  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10439  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10440  }
10441 
10442  if( !tryAddAssign( matrix_, ~rhs, row(), column() ) ) {
10443  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10444  }
10445 
10446  decltype(auto) left( derestrict( *this ) );
10447 
10448  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
10449  (~rhs).canAlias( &matrix_ ) ) {
10450  const AddType tmp( *this + (~rhs) );
10451  smpAssign( left, tmp );
10452  }
10453  else {
10454  smpAddAssign( left, ~rhs );
10455  }
10456 
10457  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10458 
10459  return *this;
10460 }
10462 //*************************************************************************************************
10463 
10464 
10465 //*************************************************************************************************
10479 template< typename MT // Type of the dense matrix
10480  , size_t... CSAs > // Compile time submatrix arguments
10481 template< typename MT2 // Type of the right-hand side matrix
10482  , bool SO > // Storage order of the right-hand side matrix
10483 inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator+=( const Matrix<MT2,SO>& rhs )
10484  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10485 {
10488  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10489 
10490  using AddType = AddTrait_t< ResultType, ResultType_t<MT2> >;
10491 
10494 
10495  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10496  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10497  }
10498 
10499  const AddType tmp( *this + (~rhs) );
10500 
10501  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10502  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10503  }
10504 
10505  decltype(auto) left( derestrict( *this ) );
10506 
10507  smpAssign( left, tmp );
10508 
10509  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10510 
10511  return *this;
10512 }
10514 //*************************************************************************************************
10515 
10516 
10517 //*************************************************************************************************
10531 template< typename MT // Type of the dense matrix
10532  , size_t... CSAs > // Compile time submatrix arguments
10533 template< typename MT2 // Type of the right-hand side matrix
10534  , bool SO > // Storage order of the right-hand side matrix
10535 inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
10536  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10537 {
10540  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10541 
10542  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
10543 
10546 
10547  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10548  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10549  }
10550 
10551  if( !trySubAssign( matrix_, ~rhs, row(), column() ) ) {
10552  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10553  }
10554 
10555  decltype(auto) left( derestrict( *this ) );
10556 
10557  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
10558  (~rhs).canAlias( &matrix_ ) ) {
10559  const SubType tmp( *this - (~rhs ) );
10560  smpAssign( left, tmp );
10561  }
10562  else {
10563  smpSubAssign( left, ~rhs );
10564  }
10565 
10566  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10567 
10568  return *this;
10569 }
10571 //*************************************************************************************************
10572 
10573 
10574 //*************************************************************************************************
10588 template< typename MT // Type of the dense matrix
10589  , size_t... CSAs > // Compile time submatrix arguments
10590 template< typename MT2 // Type of the right-hand side matrix
10591  , bool SO > // Storage order of the right-hand side matrix
10592 inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator-=( const Matrix<MT2,SO>& rhs )
10593  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10594 {
10597  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10598 
10599  using SubType = SubTrait_t< ResultType, ResultType_t<MT2> >;
10600 
10603 
10604  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10605  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10606  }
10607 
10608  const SubType tmp( *this - (~rhs) );
10609 
10610  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10611  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10612  }
10613 
10614  decltype(auto) left( derestrict( *this ) );
10615 
10616  smpAssign( left, tmp );
10617 
10618  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10619 
10620  return *this;
10621 }
10623 //*************************************************************************************************
10624 
10625 
10626 //*************************************************************************************************
10640 template< typename MT // Type of the dense matrix
10641  , size_t... CSAs > // Compile time submatrix arguments
10642 template< typename MT2 // Type of the right-hand side matrix
10643  , bool SO > // Storage order of the right-hand side matrix
10644 inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
10645  -> DisableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10646 {
10649  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10650 
10651  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
10652 
10654 
10655  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10656  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10657  }
10658 
10659  if( !trySchurAssign( matrix_, ~rhs, row(), column() ) ) {
10660  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10661  }
10662 
10663  decltype(auto) left( derestrict( *this ) );
10664 
10665  if( ( ( IsSymmetric_v<MT> || IsHermitian_v<MT> ) && hasOverlap() ) ||
10666  (~rhs).canAlias( &matrix_ ) ) {
10667  const SchurType tmp( *this % (~rhs) );
10668  if( IsSparseMatrix_v<SchurType> )
10669  reset();
10670  smpAssign( left, tmp );
10671  }
10672  else {
10673  smpSchurAssign( left, ~rhs );
10674  }
10675 
10676  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10677 
10678  return *this;
10679 }
10681 //*************************************************************************************************
10682 
10683 
10684 //*************************************************************************************************
10698 template< typename MT // Type of the dense matrix
10699  , size_t... CSAs > // Compile time submatrix arguments
10700 template< typename MT2 // Type of the right-hand side matrix
10701  , bool SO > // Storage order of the right-hand side matrix
10702 inline auto Submatrix<MT,aligned,true,true,CSAs...>::operator%=( const Matrix<MT2,SO>& rhs )
10703  -> EnableIf_t< EnforceEvaluation_v<MT,MT2>, Submatrix& >
10704 {
10707  BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType_t<MT2> );
10708 
10709  using SchurType = SchurTrait_t< ResultType, ResultType_t<MT2> >;
10710 
10712 
10713  if( rows() != (~rhs).rows() || columns() != (~rhs).columns() ) {
10714  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
10715  }
10716 
10717  const SchurType tmp( *this % (~rhs) );
10718 
10719  if( !tryAssign( matrix_, tmp, row(), column() ) ) {
10720  BLAZE_THROW_INVALID_ARGUMENT( "Invalid assignment to restricted matrix" );
10721  }
10722 
10723  decltype(auto) left( derestrict( *this ) );
10724 
10725  if( IsSparseMatrix_v<SchurType> ) {
10726  reset();
10727  }
10728 
10729  smpAssign( left, tmp );
10730 
10731  BLAZE_INTERNAL_ASSERT( isIntact( matrix_ ), "Invariant violation detected" );
10732 
10733  return *this;
10734 }
10736 //*************************************************************************************************
10737 
10738 
10739 
10740 
10741 //=================================================================================================
10742 //
10743 // UTILITY FUNCTIONS
10744 //
10745 //=================================================================================================
10746 
10747 //*************************************************************************************************
10753 template< typename MT // Type of the dense matrix
10754  , size_t... CSAs > // Compile time submatrix arguments
10755 inline MT& Submatrix<MT,aligned,true,true,CSAs...>::operand() noexcept
10756 {
10757  return matrix_;
10758 }
10760 //*************************************************************************************************
10761 
10762 
10763 //*************************************************************************************************
10769 template< typename MT // Type of the dense matrix
10770  , size_t... CSAs > // Compile time submatrix arguments
10771 inline const MT& Submatrix<MT,aligned,true,true,CSAs...>::operand() const noexcept
10772 {
10773  return matrix_;
10774 }
10776 //*************************************************************************************************
10777 
10778 
10779 //*************************************************************************************************
10788 template< typename MT // Type of the dense matrix
10789  , size_t... CSAs > // Compile time submatrix arguments
10790 inline size_t Submatrix<MT,aligned,true,true,CSAs...>::spacing() const noexcept
10791 {
10792  return matrix_.spacing();
10793 }
10795 //*************************************************************************************************
10796 
10797 
10798 //*************************************************************************************************
10804 template< typename MT // Type of the dense matrix
10805  , size_t... CSAs > // Compile time submatrix arguments
10806 inline size_t Submatrix<MT,aligned,true,true,CSAs...>::capacity() const noexcept
10807 {
10808  return rows() * columns();
10809 }
10811 //*************************************************************************************************
10812 
10813 
10814 //*************************************************************************************************
10821 template< typename MT // Type of the dense matrix
10822  , size_t... CSAs > // Compile time submatrix arguments
10823 inline size_t Submatrix<MT,aligned,true,true,CSAs...>::capacity( size_t j ) const noexcept
10824 {
10825  UNUSED_PARAMETER( j );
10826 
10827  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
10828 
10829  return rows();
10830 }
10832 //*************************************************************************************************
10833 
10834 
10835 //*************************************************************************************************
10841 template< typename MT // Type of the dense matrix
10842  , size_t... CSAs > // Compile time submatrix arguments
10844 {
10845  const size_t iend( row() + rows() );
10846  const size_t jend( column() + columns() );
10847  size_t nonzeros( 0UL );
10848 
10849  for( size_t j=column(); j<jend; ++j )
10850  for( size_t i=row(); i<iend; ++i )
10851  if( !isDefault( matrix_(i,j) ) )
10852  ++nonzeros;
10853 
10854  return nonzeros;
10855 }
10857 //*************************************************************************************************
10858 
10859 
10860 //*************************************************************************************************
10867 template< typename MT // Type of the dense matrix
10868  , size_t... CSAs > // Compile time submatrix arguments
10869 inline size_t Submatrix<MT,aligned,true,true,CSAs...>::nonZeros( size_t j ) const
10870 {
10871  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
10872 
10873  const size_t iend( row() + rows() );
10874  size_t nonzeros( 0UL );
10875 
10876  for( size_t i=row(); i<iend; ++i )
10877  if( !isDefault( matrix_(i,column()+j) ) )
10878  ++nonzeros;
10879 
10880  return nonzeros;
10881 }
10883 //*************************************************************************************************
10884 
10885 
10886 //*************************************************************************************************
10892 template< typename MT // Type of the dense matrix
10893  , size_t... CSAs > // Compile time submatrix arguments
10895 {
10896  using blaze::clear;
10897 
10898  for( size_t j=column(); j<column()+columns(); ++j )
10899  {
10900  const size_t ibegin( ( IsLower_v<MT> )
10901  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
10902  ?( max( j+1UL, row() ) )
10903  :( max( j, row() ) ) )
10904  :( row() ) );
10905  const size_t iend ( ( IsUpper_v<MT> )
10906  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
10907  ?( min( j, row()+rows() ) )
10908  :( min( j+1UL, row()+rows() ) ) )
10909  :( row()+rows() ) );
10910 
10911  for( size_t i=ibegin; i<iend; ++i )
10912  clear( matrix_(i,j) );
10913  }
10914 }
10916 //*************************************************************************************************
10917 
10918 
10919 //*************************************************************************************************
10926 template< typename MT // Type of the dense matrix
10927  , size_t... CSAs > // Compile time submatrix arguments
10928 inline void Submatrix<MT,aligned,true,true,CSAs...>::reset( size_t j )
10929 {
10930  using blaze::clear;
10931 
10932  BLAZE_USER_ASSERT( j < columns(), "Invalid column access index" );
10933 
10934  const size_t ibegin( ( IsLower_v<MT> )
10935  ?( ( IsUniLower_v<MT> || IsStrictlyLower_v<MT> )
10936  ?( max( j+1UL, row() ) )
10937  :( max( j, row() ) ) )
10938  :( row() ) );
10939  const size_t iend ( ( IsUpper_v<MT> )
10940  ?( ( IsUniUpper_v<MT> || IsStrictlyUpper_v<MT> )
10941  ?( min( j, row()+rows() ) )
10942  :( min( j+1UL, row()+rows() ) ) )
10943  :( row()+rows() ) );
10944 
10945  for( size_t i=ibegin; i<iend; ++i )
10946  clear( matrix_(i,column()+j) );
10947 }
10949 //*************************************************************************************************
10950 
10951 
10952 //*************************************************************************************************
10962 template< typename MT // Type of the dense matrix
10963  , size_t... CSAs > // Compile time submatrix arguments
10964 inline bool Submatrix<MT,aligned,true,true,CSAs...>::hasOverlap() const noexcept
10965 {
10966  BLAZE_INTERNAL_ASSERT( IsSymmetric_v<MT> || IsHermitian_v<MT>, "Invalid matrix detected" );
10967 
10968  if( ( row() + rows() <= column() ) || ( column() + columns() <= row() ) )
10969  return false;
10970  else return true;
10971 }
10973 //*************************************************************************************************
10974 
10975 
10976 
10977 
10978 //=================================================================================================
10979 //
10980 // NUMERIC FUNCTIONS
10981 //
10982 //=================================================================================================
10983 
10984 //*************************************************************************************************
11002 template< typename MT // Type of the dense matrix
11003  , size_t... CSAs > // Compile time submatrix arguments
11004 inline Submatrix<MT,aligned,true,true,CSAs...>&
11006 {
11007  if( rows() != columns() ) {
11008  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
11009  }
11010 
11011  if( !tryAssign( matrix_, trans( *this ), row(), column() ) ) {
11012  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
11013  }
11014 
11015  decltype(auto) left( derestrict( *this ) );
11016  const ResultType tmp( trans( *this ) );
11017 
11018  smpAssign( left, tmp );
11019 
11020  return *this;
11021 }
11023 //*************************************************************************************************
11024 
11025 
11026 //*************************************************************************************************
11044 template< typename MT // Type of the dense matrix
11045  , size_t... CSAs > // Compile time submatrix arguments
11046 inline Submatrix<MT,aligned,true,true,CSAs...>&
11047  Submatrix<MT,aligned,true,true,CSAs...>::ctranspose()
11048 {
11049  if( rows() != columns() ) {
11050  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose of a non-quadratic submatrix" );
11051  }
11052 
11053  if( !tryAssign( matrix_, ctrans( *this ), row(), column() ) ) {
11054  BLAZE_THROW_LOGIC_ERROR( "Invalid transpose operation" );
11055  }
11056 
11057  decltype(auto) left( derestrict( *this ) );
11058  const ResultType tmp( ctrans( *this ) );
11059 
11060  smpAssign( left, tmp );
11061 
11062  return *this;
11063 }
11065 //*************************************************************************************************
11066 
11067 
11068 //*************************************************************************************************
11081 template< typename MT // Type of the dense matrix
11082  , size_t... CSAs > // Compile time submatrix arguments
11083 template< typename Other > // Data type of the scalar value
11084 inline Submatrix<MT,aligned,true,true,CSAs...>&
11085  Submatrix<MT,aligned,true,true,CSAs...>::scale( const Other& scalar )
11086 {
11088 
11089  const size_t jend( column() + columns() );
11090 
11091  for( size_t j=column(); j<jend; ++j )
11092  {
11093  const size_t ibegin( ( IsLower_v<MT> )
11094  ?( ( IsStrictlyLower_v<MT> )
11095  ?( max( j+1UL, row() ) )
11096  :( max( j, row() ) ) )
11097  :( row() ) );
11098  const size_t iend ( ( IsUpper_v<MT> )
11099  ?( ( IsStrictlyUpper_v<MT> )
11100  ?( min( j, row()+rows() ) )
11101  :( min( j+1UL, row()+rows() ) ) )
11102  :( row()+rows() ) );
11103 
11104  for( size_t i=ibegin; i<iend; ++i )
11105  matrix_(i,j) *= scalar;
11106  }
11107 
11108  return *this;
11109 }
11111 //*************************************************************************************************
11112 
11113 
11114 
11115 
11116 //=================================================================================================
11117 //
11118 // EXPRESSION TEMPLATE EVALUATION FUNCTIONS
11119 //
11120 //=================================================================================================
11121 
11122 //*************************************************************************************************
11133 template< typename MT // Type of the dense matrix
11134  , size_t... CSAs > // Compile time submatrix arguments
11135 template< typename Other > // Data type of the foreign expression
11136 inline bool Submatrix<MT,aligned,true,true,CSAs...>::canAlias( const Other* alias ) const noexcept
11137 {
11138  return matrix_.isAliased( alias );
11139 }
11141 //*************************************************************************************************
11142 
11143 
11144 //*************************************************************************************************
11155 template< typename MT // Type of the dense matrix
11156  , size_t... CSAs > // Compile time submatrix arguments
11157 template< typename MT2 // Data type of the foreign dense submatrix
11158  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
11159  , bool SO2 // Storage order of the foreign dense submatrix
11160  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
11161 inline bool
11162  Submatrix<MT,aligned,true,true,CSAs...>::canAlias( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
11163 {
11164  return ( matrix_.isAliased( &alias->matrix_ ) &&
11165  ( row() + rows() > alias->row() ) &&
11166  ( row() < alias->row() + alias->rows() ) &&
11167  ( column() + columns() > alias->column() ) &&
11168  ( column() < alias->column() + alias->columns() ) );
11169 }
11171 //*************************************************************************************************
11172 
11173 
11174 //*************************************************************************************************
11185 template< typename MT // Type of the dense matrix
11186  , size_t... CSAs > // Compile time submatrix arguments
11187 template< typename Other > // Data type of the foreign expression
11188 inline bool Submatrix<MT,aligned,true,true,CSAs...>::isAliased( const Other* alias ) const noexcept
11189 {
11190  return matrix_.isAliased( alias );
11191 }
11193 //*************************************************************************************************
11194 
11195 
11196 //*************************************************************************************************
11207 template< typename MT // Type of the dense matrix
11208  , size_t... CSAs > // Compile time submatrix arguments
11209 template< typename MT2 // Data type of the foreign dense submatrix
11210  , AlignmentFlag AF2 // Alignment flag of the foreign dense submatrix
11211  , bool SO2 // Storage order of the foreign dense submatrix
11212  , size_t... CSAs2 > // Compile time submatrix arguments of the foreign dense submatrix
11213 inline bool
11214  Submatrix<MT,aligned,true,true,CSAs...>::isAliased( const Submatrix<MT2,AF2,SO2,true,CSAs2...>* alias ) const noexcept
11215 {
11216  return ( matrix_.isAliased( &alias->matrix_ ) &&
11217  ( row() + rows() > alias->row() ) &&
11218  ( row() < alias->row() + alias->rows() ) &&
11219  ( column() + columns() > alias->column() ) &&
11220  ( column() < alias->column() + alias->columns() ) );
11221 }
11223 //*************************************************************************************************
11224 
11225 
11226 //*************************************************************************************************
11236 template< typename MT // Type of the dense matrix
11237  , size_t... CSAs > // Compile time submatrix arguments
11238 inline bool Submatrix<MT,aligned,true,true,CSAs...>::isAligned() const noexcept
11239 {
11240  return true;
11241 }
11243 //*************************************************************************************************
11244 
11245 
11246 //*************************************************************************************************
11257 template< typename MT // Type of the dense matrix
11258  , size_t... CSAs > // Compile time submatrix arguments
11259 inline bool Submatrix<MT,aligned,true,true,CSAs...>::canSMPAssign() const noexcept
11260 {
11261  return ( rows() * columns() >= SMP_DMATASSIGN_THRESHOLD );
11262 }
11264 //*************************************************************************************************
11265 
11266 
11267 //*************************************************************************************************
11282 template< typename MT // Type of the dense matrix
11283  , size_t... CSAs > // Compile time submatrix arguments
11284 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,true,true,CSAs...>::SIMDType
11285  Submatrix<MT,aligned,true,true,CSAs...>::load( size_t i, size_t j ) const noexcept
11286 {
11287  return loada( i, j );
11288 }
11290 //*************************************************************************************************
11291 
11292 
11293 //*************************************************************************************************
11308 template< typename MT // Type of the dense matrix
11309  , size_t... CSAs > // Compile time submatrix arguments
11310 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,true,true,CSAs...>::SIMDType
11311  Submatrix<MT,aligned,true,true,CSAs...>::loada( size_t i, size_t j ) const noexcept
11312 {
11314 
11315  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11316  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11317  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11318  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11319 
11320  return matrix_.loada( row()+i, column()+j );
11321 }
11323 //*************************************************************************************************
11324 
11325 
11326 //*************************************************************************************************
11341 template< typename MT // Type of the dense matrix
11342  , size_t... CSAs > // Compile time submatrix arguments
11343 BLAZE_ALWAYS_INLINE typename Submatrix<MT,aligned,true,true,CSAs...>::SIMDType
11344  Submatrix<MT,aligned,true,true,CSAs...>::loadu( size_t i, size_t j ) const noexcept
11345 {
11347 
11348  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11349  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11350  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11351  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11352 
11353  return matrix_.loadu( row()+i, column()+j );
11354 }
11356 //*************************************************************************************************
11357 
11358 
11359 //*************************************************************************************************
11375 template< typename MT // Type of the dense matrix
11376  , size_t... CSAs > // Compile time submatrix arguments
11378  Submatrix<MT,aligned,true,true,CSAs...>::store( size_t i, size_t j, const SIMDType& value ) noexcept
11379 {
11380  storea( i, j, value );
11381 }
11383 //*************************************************************************************************
11384 
11385 
11386 //*************************************************************************************************
11402 template< typename MT // Type of the dense matrix
11403  , size_t... CSAs > // Compile time submatrix arguments
11405  Submatrix<MT,aligned,true,true,CSAs...>::storea( size_t i, size_t j, const SIMDType& value ) noexcept
11406 {
11408 
11409  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11410  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11411  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11412  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11413 
11414  matrix_.storea( row()+i, column()+j, value );
11415 }
11417 //*************************************************************************************************
11418 
11419 
11420 //*************************************************************************************************
11436 template< typename MT // Type of the dense matrix
11437  , size_t... CSAs > // Compile time submatrix arguments
11439  Submatrix<MT,aligned,true,true,CSAs...>::storeu( size_t i, size_t j, const SIMDType& value ) noexcept
11440 {
11442 
11443  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11444  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11445  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11446  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11447 
11448  matrix_.storeu( row()+i, column()+j, value );
11449 }
11451 //*************************************************************************************************
11452 
11453 
11454 //*************************************************************************************************
11471 template< typename MT // Type of the dense matrix
11472  , size_t... CSAs > // Compile time submatrix arguments
11474  Submatrix<MT,aligned,true,true,CSAs...>::stream( size_t i, size_t j, const SIMDType& value ) noexcept
11475 {
11477 
11478  BLAZE_INTERNAL_ASSERT( i < rows(), "Invalid row access index" );
11479  BLAZE_INTERNAL_ASSERT( i + SIMDSIZE <= rows(), "Invalid row access index" );
11480  BLAZE_INTERNAL_ASSERT( i % SIMDSIZE == 0UL, "Invalid row access index" );
11481  BLAZE_INTERNAL_ASSERT( j < columns(), "Invalid column access index" );
11482 
11483  matrix_.stream( row()+i, column()+j, value );
11484 }
11486 //*************************************************************************************************
11487 
11488 
11489 //*************************************************************************************************
11501 template< typename MT // Type of the dense matrix
11502  , size_t... CSAs > // Compile time submatrix arguments
11503 template< typename MT2 > // Type of the right-hand side dense matrix
11504 inline auto Submatrix<MT,aligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
11505  -> DisableIf_t< VectorizedAssign_v<MT2> >
11506 {
11507  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11508  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11509 
11510  const size_t ipos( rows() & size_t(-2) );
11511  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
11512 
11513  for( size_t j=0UL; j<columns(); ++j ) {
11514  for( size_t i=0UL; i<ipos; i+=2UL ) {
11515  matrix_(row()+i ,column()+j) = (~rhs)(i ,j);
11516  matrix_(row()+i+1UL,column()+j) = (~rhs)(i+1UL,j);
11517  }
11518  if( ipos < rows() ) {
11519  matrix_(row()+ipos,column()+j) = (~rhs)(ipos,j);
11520  }
11521  }
11522 }
11524 //*************************************************************************************************
11525 
11526 
11527 //*************************************************************************************************
11539 template< typename MT // Type of the dense matrix
11540  , size_t... CSAs > // Compile time submatrix arguments
11541 template< typename MT2 > // Type of the right-hand side dense matrix
11542 inline auto Submatrix<MT,aligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,true>& rhs )
11543  -> EnableIf_t< VectorizedAssign_v<MT2> >
11544 {
11546 
11547  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11548  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11549 
11550  const size_t ipos( rows() & size_t(-SIMDSIZE) );
11551  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
11552 
11553  if( useStreaming &&
11554  rows()*columns() > ( cacheSize / ( sizeof(ElementType) * 3UL ) ) &&
11555  !(~rhs).isAliased( &matrix_ ) )
11556  {
11557  for( size_t j=0UL; j<columns(); ++j )
11558  {
11559  size_t i( 0UL );
11560  Iterator left( begin(j) );
11561  ConstIterator_t<MT2> right( (~rhs).begin(j) );
11562 
11563  for( ; i<ipos; i+=SIMDSIZE ) {
11564  left.stream( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11565  }
11566  for( ; i<rows(); ++i ) {
11567  *left = *right; ++left; ++right;
11568  }
11569  }
11570  }
11571  else
11572  {
11573  for( size_t j=0UL; j<columns(); ++j )
11574  {
11575  size_t i( 0UL );
11576  Iterator left( begin(j) );
11577  ConstIterator_t<MT2> right( (~rhs).begin(j) );
11578 
11579  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
11580  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11581  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11582  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11583  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11584  }
11585  for( ; i<ipos; i+=SIMDSIZE ) {
11586  left.store( right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11587  }
11588  for( ; i<rows(); ++i ) {
11589  *left = *right; ++left; ++right;
11590  }
11591  }
11592  }
11593 }
11595 //*************************************************************************************************
11596 
11597 
11598 //*************************************************************************************************
11610 template< typename MT // Type of the dense matrix
11611  , size_t... CSAs > // Compile time submatrix arguments
11612 template< typename MT2 > // Type of the right-hand side dense matrix
11613 inline void Submatrix<MT,aligned,true,true,CSAs...>::assign( const DenseMatrix<MT2,false>& rhs )
11614 {
11616 
11617  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11618  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11619 
11620  constexpr size_t block( BLOCK_SIZE );
11621 
11622  for( size_t jj=0UL; jj<columns(); jj+=block ) {
11623  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
11624  for( size_t ii=0UL; ii<rows(); ii+=block ) {
11625  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
11626  for( size_t j=jj; j<jend; ++j ) {
11627  for( size_t i=ii; i<iend; ++i ) {
11628  matrix_(row()+i,column()+j) = (~rhs)(i,j);
11629  }
11630  }
11631  }
11632  }
11633 }
11635 //*************************************************************************************************
11636 
11637 
11638 //*************************************************************************************************
11650 template< typename MT // Type of the dense matrix
11651  , size_t... CSAs > // Compile time submatrix arguments
11652 template< typename MT2 > // Type of the right-hand side sparse matrix
11653 inline void Submatrix<MT,aligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,true>& rhs )
11654 {
11655  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11656  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11657 
11658  for( size_t j=0UL; j<columns(); ++j )
11659  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
11660  matrix_(row()+element->index(),column()+j) = element->value();
11661 }
11663 //*************************************************************************************************
11664 
11665 
11666 //*************************************************************************************************
11678 template< typename MT // Type of the dense matrix
11679  , size_t... CSAs > // Compile time submatrix arguments
11680 template< typename MT2 > // Type of the right-hand side sparse matrix
11681 inline void Submatrix<MT,aligned,true,true,CSAs...>::assign( const SparseMatrix<MT2,false>& rhs )
11682 {
11684 
11685  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11686  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11687 
11688  for( size_t i=0UL; i<rows(); ++i )
11689  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
11690  matrix_(row()+i,column()+element->index()) = element->value();
11691 }
11693 //*************************************************************************************************
11694 
11695 
11696 //*************************************************************************************************
11708 template< typename MT // Type of the dense matrix
11709  , size_t... CSAs > // Compile time submatrix arguments
11710 template< typename MT2 > // Type of the right-hand side dense matrix
11711 inline auto Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
11712  -> DisableIf_t< VectorizedAddAssign_v<MT2> >
11713 {
11714  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11715  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11716 
11717  const size_t ipos( rows() & size_t(-2) );
11718  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
11719 
11720  for( size_t j=0UL; j<columns(); ++j )
11721  {
11722  if( IsDiagonal_v<MT2> ) {
11723  matrix_(row()+j,column()+j) += (~rhs)(j,j);
11724  }
11725  else {
11726  for( size_t i=0UL; i<ipos; i+=2UL ) {
11727  matrix_(row()+i ,column()+j) += (~rhs)(i ,j);
11728  matrix_(row()+i+1UL,column()+j) += (~rhs)(i+1UL,j);
11729  }
11730  if( ipos < rows() ) {
11731  matrix_(row()+ipos,column()+j) += (~rhs)(ipos,j);
11732  }
11733  }
11734  }
11735 }
11737 //*************************************************************************************************
11738 
11739 
11740 //*************************************************************************************************
11752 template< typename MT // Type of the dense matrix
11753  , size_t... CSAs > // Compile time submatrix arguments
11754 template< typename MT2 > // Type of the right-hand side dense matrix
11755 inline auto Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,true>& rhs )
11756  -> EnableIf_t< VectorizedAddAssign_v<MT2> >
11757 {
11759 
11760  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11761  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11762 
11763  for( size_t j=0UL; j<columns(); ++j )
11764  {
11765  const size_t ibegin( ( IsLower_v<MT> )
11766  ?( ( IsStrictlyLower_v<MT> ? j+1UL : j ) & size_t(-SIMDSIZE) )
11767  :( 0UL ) );
11768  const size_t iend ( ( IsUpper_v<MT> )
11769  ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
11770  :( rows() ) );
11771  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
11772 
11773  const size_t ipos( iend & size_t(-SIMDSIZE) );
11774  BLAZE_INTERNAL_ASSERT( ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
11775 
11776  size_t i( ibegin );
11777  Iterator left( begin(j) + ibegin );
11778  ConstIterator_t<MT2> right( (~rhs).begin(j) + ibegin );
11779 
11780  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
11781  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11782  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11783  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11784  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11785  }
11786  for( ; i<ipos; i+=SIMDSIZE ) {
11787  left.store( left.load() + right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11788  }
11789  for( ; i<iend; ++i ) {
11790  *left += *right; ++left; ++right;
11791  }
11792  }
11793 }
11795 //*************************************************************************************************
11796 
11797 
11798 //*************************************************************************************************
11810 template< typename MT // Type of the dense matrix
11811  , size_t... CSAs > // Compile time submatrix arguments
11812 template< typename MT2 > // Type of the right-hand side dense matrix
11813 inline void Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const DenseMatrix<MT2,false>& rhs )
11814 {
11816 
11817  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11818  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11819 
11820  constexpr size_t block( BLOCK_SIZE );
11821 
11822  for( size_t jj=0UL; jj<columns(); jj+=block ) {
11823  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
11824  for( size_t ii=0UL; ii<rows(); ii+=block ) {
11825  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
11826  for( size_t j=jj; j<jend; ++j ) {
11827  for( size_t i=ii; i<iend; ++i ) {
11828  matrix_(row()+i,column()+j) += (~rhs)(i,j);
11829  }
11830  }
11831  }
11832  }
11833 }
11835 //*************************************************************************************************
11836 
11837 
11838 //*************************************************************************************************
11850 template< typename MT // Type of the dense matrix
11851  , size_t... CSAs > // Compile time submatrix arguments
11852 template< typename MT2 > // Type of the right-hand side sparse matrix
11853 inline void Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,true>& rhs )
11854 {
11855  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11856  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11857 
11858  for( size_t j=0UL; j<columns(); ++j )
11859  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
11860  matrix_(row()+element->index(),column()+j) += element->value();
11861 }
11863 //*************************************************************************************************
11864 
11865 
11866 //*************************************************************************************************
11878 template< typename MT // Type of the dense matrix
11879  , size_t... CSAs > // Compile time submatrix arguments
11880 template< typename MT2 > // Type of the right-hand side sparse matrix
11881 inline void Submatrix<MT,aligned,true,true,CSAs...>::addAssign( const SparseMatrix<MT2,false>& rhs )
11882 {
11884 
11885  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11886  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11887 
11888  for( size_t i=0UL; i<rows(); ++i )
11889  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
11890  matrix_(row()+i,column()+element->index()) += element->value();
11891 }
11893 //*************************************************************************************************
11894 
11895 
11896 //*************************************************************************************************
11908 template< typename MT // Type of the dense matrix
11909  , size_t... CSAs > // Compile time submatrix arguments
11910 template< typename MT2 > // Type of the right-hand side dense matrix
11911 inline auto Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
11912  -> DisableIf_t< VectorizedSubAssign_v<MT2> >
11913 {
11914  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11915  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11916 
11917  const size_t ipos( rows() & size_t(-2) );
11918  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
11919 
11920  for( size_t j=0UL; j<columns(); ++j )
11921  {
11922  if( IsDiagonal_v<MT2> ) {
11923  matrix_(row()+j,column()+j) -= (~rhs)(j,j);
11924  }
11925  else {
11926  for( size_t i=0UL; i<ipos; i+=2UL ) {
11927  matrix_(row()+i ,column()+j) -= (~rhs)(i ,j);
11928  matrix_(row()+i+1UL,column()+j) -= (~rhs)(i+1UL,j);
11929  }
11930  if( ipos < rows() ) {
11931  matrix_(row()+ipos,column()+j) -= (~rhs)(ipos,j);
11932  }
11933  }
11934  }
11935 }
11937 //*************************************************************************************************
11938 
11939 
11940 //*************************************************************************************************
11952 template< typename MT // Type of the dense matrix
11953  , size_t... CSAs > // Compile time submatrix arguments
11954 template< typename MT2 > // Type of the right-hand side dense matrix
11955 inline auto Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,true>& rhs )
11956  -> EnableIf_t< VectorizedSubAssign_v<MT2> >
11957 {
11959 
11960  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
11961  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
11962 
11963  for( size_t j=0UL; j<columns(); ++j )
11964  {
11965  const size_t ibegin( ( IsLower_v<MT> )
11966  ?( ( IsStrictlyLower_v<MT> ? j+1UL : j ) & size_t(-SIMDSIZE) )
11967  :( 0UL ) );
11968  const size_t iend ( ( IsUpper_v<MT> )
11969  ?( IsStrictlyUpper_v<MT> ? j : j+1UL )
11970  :( rows() ) );
11971  BLAZE_INTERNAL_ASSERT( ibegin <= iend, "Invalid loop indices detected" );
11972 
11973  const size_t ipos( iend & size_t(-SIMDSIZE) );
11974  BLAZE_INTERNAL_ASSERT( ( iend - ( iend % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
11975 
11976  size_t i( ibegin );
11977  Iterator left( begin(j) + ibegin );
11978  ConstIterator_t<MT2> right( (~rhs).begin(j) + ibegin );
11979 
11980  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
11981  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11982  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11983  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11984  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11985  }
11986  for( ; i<ipos; i+=SIMDSIZE ) {
11987  left.store( left.load() - right.load() ); left += SIMDSIZE; right += SIMDSIZE;
11988  }
11989  for( ; i<iend; ++i ) {
11990  *left -= *right; ++left; ++right;
11991  }
11992  }
11993 }
11995 //*************************************************************************************************
11996 
11997 
11998 //*************************************************************************************************
12010 template< typename MT // Type of the dense matrix
12011  , size_t... CSAs > // Compile time submatrix arguments
12012 template< typename MT2 > // Type of the right-hand side dense matrix
12013 inline void Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const DenseMatrix<MT2,false>& rhs )
12014 {
12016 
12017  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12018  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12019 
12020  constexpr size_t block( BLOCK_SIZE );
12021 
12022  for( size_t jj=0UL; jj<columns(); jj+=block ) {
12023  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
12024  for( size_t ii=0UL; ii<rows(); ii+=block ) {
12025  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
12026  for( size_t j=jj; j<jend; ++j ) {
12027  for( size_t i=ii; i<iend; ++i ) {
12028  matrix_(row()+i,column()+j) -= (~rhs)(i,j);
12029  }
12030  }
12031  }
12032  }
12033 }
12035 //*************************************************************************************************
12036 
12037 
12038 //*************************************************************************************************
12050 template< typename MT // Type of the dense matrix
12051  , size_t... CSAs > // Compile time submatrix arguments
12052 template< typename MT2 > // Type of the right-hand side sparse matrix
12053 inline void Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,true>& rhs )
12054 {
12055  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12056  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12057 
12058  for( size_t j=0UL; j<columns(); ++j )
12059  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element )
12060  matrix_(row()+element->index(),column()+j) -= element->value();
12061 }
12063 //*************************************************************************************************
12064 
12065 
12066 //*************************************************************************************************
12078 template< typename MT // Type of the dense matrix
12079  , size_t... CSAs > // Compile time submatrix arguments
12080 template< typename MT2 > // Type of the right-hand side sparse matrix
12081 inline void Submatrix<MT,aligned,true,true,CSAs...>::subAssign( const SparseMatrix<MT2,false>& rhs )
12082 {
12084 
12085  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12086  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12087 
12088  for( size_t i=0UL; i<rows(); ++i )
12089  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element )
12090  matrix_(row()+i,column()+element->index()) -= element->value();
12091 }
12093 //*************************************************************************************************
12094 
12095 
12096 //*************************************************************************************************
12108 template< typename MT // Type of the dense matrix
12109  , size_t... CSAs > // Compile time submatrix arguments
12110 template< typename MT2 > // Type of the right-hand side dense matrix
12111 inline auto Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
12112  -> DisableIf_t< VectorizedSchurAssign_v<MT2> >
12113 {
12114  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12115  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12116 
12117  const size_t ipos( rows() & size_t(-2) );
12118  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % 2UL ) ) == ipos, "Invalid end calculation" );
12119 
12120  for( size_t j=0UL; j<columns(); ++j ) {
12121  for( size_t i=0UL; i<ipos; i+=2UL ) {
12122  matrix_(row()+i ,column()+j) *= (~rhs)(i ,j);
12123  matrix_(row()+i+1UL,column()+j) *= (~rhs)(i+1UL,j);
12124  }
12125  if( ipos < rows() ) {
12126  matrix_(row()+ipos,column()+j) *= (~rhs)(ipos,j);
12127  }
12128  }
12129 }
12131 //*************************************************************************************************
12132 
12133 
12134 //*************************************************************************************************
12147 template< typename MT // Type of the dense matrix
12148  , size_t... CSAs > // Compile time submatrix arguments
12149 template< typename MT2 > // Type of the right-hand side dense matrix
12150 inline auto Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,true>& rhs )
12151  -> EnableIf_t< VectorizedSchurAssign_v<MT2> >
12152 {
12154 
12155  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12156  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12157 
12158  for( size_t j=0UL; j<columns(); ++j )
12159  {
12160  const size_t ipos( rows() & size_t(-SIMDSIZE) );
12161  BLAZE_INTERNAL_ASSERT( ( rows() - ( rows() % (SIMDSIZE) ) ) == ipos, "Invalid end calculation" );
12162 
12163  size_t i( 0UL );
12164  Iterator left( begin(j) );
12165  ConstIterator_t<MT2> right( (~rhs).begin(j) );
12166 
12167  for( ; (i+SIMDSIZE*3UL) < ipos; i+=SIMDSIZE*4UL ) {
12168  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12169  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12170  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12171  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12172  }
12173  for( ; i<ipos; i+=SIMDSIZE ) {
12174  left.store( left.load() * right.load() ); left += SIMDSIZE; right += SIMDSIZE;
12175  }
12176  for( ; i<rows(); ++i ) {
12177  *left *= *right; ++left; ++right;
12178  }
12179  }
12180 }
12182 //*************************************************************************************************
12183 
12184 
12185 //*************************************************************************************************
12197 template< typename MT // Type of the dense matrix
12198  , size_t... CSAs > // Compile time submatrix arguments
12199 template< typename MT2 > // Type of the right-hand side dense matrix
12200 inline void Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const DenseMatrix<MT2,false>& rhs )
12201 {
12203 
12204  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12205  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12206 
12207  constexpr size_t block( BLOCK_SIZE );
12208 
12209  for( size_t jj=0UL; jj<columns(); jj+=block ) {
12210  const size_t jend( ( columns()<(jj+block) )?( columns() ):( jj+block ) );
12211  for( size_t ii=0UL; ii<rows(); ii+=block ) {
12212  const size_t iend( ( rows()<(ii+block) )?( rows() ):( ii+block ) );
12213  for( size_t j=jj; j<jend; ++j ) {
12214  for( size_t i=ii; i<iend; ++i ) {
12215  matrix_(row()+i,column()+j) *= (~rhs)(i,j);
12216  }
12217  }
12218  }
12219  }
12220 }
12222 //*************************************************************************************************
12223 
12224 
12225 //*************************************************************************************************
12237 template< typename MT // Type of the dense matrix
12238  , size_t... CSAs > // Compile time submatrix arguments
12239 template< typename MT2 > // Type of the right-hand side sparse matrix
12240 inline void Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,true>& rhs )
12241 {
12242  using blaze::reset;
12243 
12244  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12245  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12246 
12247  for( size_t j=0UL; j<columns(); ++j )
12248  {
12249  size_t i( 0UL );
12250 
12251  for( ConstIterator_t<MT2> element=(~rhs).begin(j); element!=(~rhs).end(j); ++element ) {
12252  for( ; i<element->index(); ++i )
12253  reset( matrix_(row()+i,column()+j) );
12254  matrix_(row()+i,column()+j) *= element->value();
12255  ++i;
12256  }
12257 
12258  for( ; i<rows(); ++i ) {
12259  reset( matrix_(row()+i,column()+j) );
12260  }
12261  }
12262 }
12264 //*************************************************************************************************
12265 
12266 
12267 //*************************************************************************************************
12279 template< typename MT // Type of the dense matrix
12280  , size_t... CSAs > // Compile time submatrix arguments
12281 template< typename MT2 > // Type of the right-hand side sparse matrix
12282 inline void Submatrix<MT,aligned,true,true,CSAs...>::schurAssign( const SparseMatrix<MT2,false>& rhs )
12283 {
12284  using blaze::reset;
12285 
12287 
12288  BLAZE_INTERNAL_ASSERT( rows() == (~rhs).rows() , "Invalid number of rows" );
12289  BLAZE_INTERNAL_ASSERT( columns() == (~rhs).columns(), "Invalid number of columns" );
12290 
12291  for( size_t i=0UL; i<rows(); ++i )
12292  {
12293  size_t j( 0UL );
12294 
12295  for( ConstIterator_t<MT2> element=(~rhs).begin(i); element!=(~rhs).end(i); ++element ) {
12296  for( ; j<element->index(); ++j )
12297  reset( matrix_(row()+i,column()+j) );
12298  matrix_(row()+i,column()+j) *= element->value();
12299  ++j;
12300  }
12301 
12302  for( ; j<columns(); ++j ) {
12303  reset( matrix_(row()+i,column()+j) );
12304  }
12305  }
12306 }
12308 //*************************************************************************************************
12309 
12310 } // namespace blaze
12311 
12312 #endif
AlignmentFlag
Alignment flag for (un-)aligned vectors and matrices.Via these flags it is possible to specify subvec...
Definition: AlignmentFlag.h:62
Constraint on the data type.
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Constraint on the data type.
Header file for auxiliary alias declarations.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:133
Headerfile for the generic min algorithm.
Header file for the blaze::checked and blaze::unchecked instances.
constexpr bool IsContiguous_v
Auxiliary variable template for the IsContiguous type trait.The IsContiguous_v variable template prov...
Definition: IsContiguous.h:145
Header file for kernel specific block sizes.
CompressedMatrix< Type, false > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: CompressedMatrix.h:3078
Header file for the Schur product trait.
#define BLAZE_USER_ASSERT(expr, msg)
Run time assertion macro for user checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_USER_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERT flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:117
constexpr bool IsSparseMatrix_v
Auxiliary variable template for the IsSparseMatrix type trait.The IsSparseMatrix_v variable template ...
Definition: IsSparseMatrix.h:139
Header file for the alignment flag values.
Header file for the UNUSED_PARAMETER function template.
Header file for the IsUniUpper type trait.
Header file for the subtraction trait.
size_t capacity(const Matrix< MT, SO > &matrix) noexcept
Returns the maximum capacity of the matrix.
Definition: Matrix.h:546
Header file for basic type definitions.
constexpr const DenseIterator< Type, AF > operator-(const DenseIterator< Type, AF > &it, ptrdiff_t inc) noexcept
Subtraction between a DenseIterator and an integral value.
Definition: DenseIterator.h:750
MT::ElementType * data(DenseMatrix< MT, SO > &dm) noexcept
Low-level data access to the dense matrix elements.
Definition: DenseMatrix.h:169
Header file for the View base class.
constexpr bool HasSIMDSub_v
Auxiliary variable template for the HasSIMDSub type trait.The HasSIMDSub_v variable template provides...
Definition: HasSIMDSub.h:188
Header file for the IsSparseMatrix type trait.
Header file for the IsDiagonal type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_COMPUTATION_TYPE(T)
Constraint on the data type.In case the given data type T is a computational expression (i...
Definition: Computation.h:81
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
constexpr bool operator<(const NegativeAccuracy< A > &lhs, const T &rhs)
Less-than comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:332
BLAZE_ALWAYS_INLINE const EnableIf_t< IsIntegral_v< T > &&HasSize_v< T, 1UL >, If_t< IsSigned_v< T >, SIMDint8, SIMDuint8 > > loadu(const T *address) noexcept
Loads a vector of 1-byte integral values.
Definition: Loadu.h:76
This ResultType
Result type for expression template evaluations.
Definition: CompressedMatrix.h:3077
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:372
CompressedMatrix< Type, true > This
Type of this CompressedMatrix instance.
Definition: CompressedMatrix.h:3075
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:591
static constexpr bool smpAssignable
Compilation flag for SMP assignments.
Definition: CompressedMatrix.h:3113
CompressedMatrix< Type, false > TransposeType
Transpose type for expression template evaluations.
Definition: CompressedMatrix.h:3079
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SUBMATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a submatrix type (i.e. a dense or sparse submatrix), a compilation error is created.
Definition: Submatrix.h:81
void ctranspose(Matrix< MT, SO > &matrix)
In-place conjugate transpose of the given matrix.
Definition: Matrix.h:851
#define BLAZE_CONSTRAINT_MUST_NOT_BE_UNITRIANGULAR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a lower or upper unitriangular matrix ty...
Definition: UniTriangular.h:81
constexpr bool IsHermitian_v
Auxiliary variable template for the IsHermitian type trait.The IsHermitian_v variable template provid...
Definition: IsHermitian.h:171
const Type & ConstReference
Reference to a constant matrix value.
Definition: CompressedMatrix.h:3084
constexpr bool IsReference_v
Auxiliary variable template for the IsReference type trait.The IsReference_v variable template provid...
Definition: IsReference.h:95
size_t nonZeros(const Matrix< MT, SO > &matrix)
Returns the total number of non-zero elements in the matrix.
Definition: Matrix.h:584
Header file for the reset shim.
Element * Iterator
Iterator over non-constant elements.
Definition: CompressedMatrix.h:3085
Constraints on the storage order of matrix types.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_TRANSEXPR_TYPE(T)
Constraint on the data type.In case the given data type T is a transposition expression (i...
Definition: TransExpr.h:81
BLAZE_ALWAYS_INLINE EnableIf_t< IsIntegral_v< T1 > &&HasSize_v< T1, 1UL > > storeu(T1 *address, const SIMDi8< T2 > &value) noexcept
Unaligned store of a vector of 1-byte integral values.
Definition: Storeu.h:75
Header file for the RequiresEvaluation type trait.
Header file for the extended initializer_list functionality.
System settings for performance optimizations.
constexpr void UNUSED_PARAMETER(const Args &...)
Suppression of unused parameter warnings.
Definition: Unused.h:81
Header file for the IsUniLower type trait.
typename SubmatrixTrait< MT, CSAs... >::Type SubmatrixTrait_t
Auxiliary alias declaration for the SubmatrixTrait type trait.The SubmatrixTrait_t alias declaration ...
Definition: SubmatrixTrait.h:171
constexpr size_t columns(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of columns of the matrix.
Definition: Matrix.h:514
MT::ConstIterator cend(const Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:482
MT::ConstIterator cbegin(const Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:416
typename T::ElementType ElementType_t
Alias declaration for nested ElementType type definitions.The ElementType_t alias declaration provide...
Definition: Aliases.h:170
Constraint on the data type.
size_t spacing(const DenseMatrix< MT, SO > &dm) noexcept
Returns the spacing between the beginning of two rows/columns.
Definition: DenseMatrix.h:252
Header file for the matrix storage order types.
Constraint on the data type.
Header file for the implementation of a matrix representation of an initializer list.
typename EnableIf< Condition, T >::Type EnableIf_t
Auxiliary type for the EnableIf class template.The EnableIf_t alias declaration provides a convenient...
Definition: EnableIf.h:138
Headerfile for the generic max algorithm.
Header file for the DisableIf class template.
Header file for the IsStrictlyUpper type trait.
constexpr bool IsSIMDCombinable_v
Auxiliary variable template for the IsSIMDCombinable type trait.The IsSIMDCombinable_v variable templ...
Definition: IsSIMDCombinable.h:137
Header file for the IsSymmetric type trait.
MatrixAccessProxy< This > Reference
Reference to a non-constant matrix value.
Definition: CompressedMatrix.h:3083
Namespace of the Blaze C++ math library.
Definition: Blaze.h:58
#define BLAZE_ALWAYS_INLINE
Platform dependent setup of an enforced inline keyword.
Definition: Inline.h:85
decltype(auto) ctrans(const DenseMatrix< MT, SO > &dm)
Returns the conjugate transpose matrix of dm.
Definition: DMatMapExpr.h:1364
Header file for the If class template.
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
constexpr bool HasSIMDMult_v
Auxiliary variable template for the HasSIMDMult type trait.The HasSIMDMult_v variable template provid...
Definition: HasSIMDMult.h:189
SparseMatrix< This, true > BaseType
Base type of this CompressedMatrix instance.
Definition: CompressedMatrix.h:3076
#define BLAZE_CONSTRAINT_MUST_NOT_BE_POINTER_TYPE(T)
Constraint on the data type.In case the given data type T is not a pointer type, a compilation error ...
Definition: Pointer.h:79
Type ElementType
Type of the compressed matrix elements.
Definition: CompressedMatrix.h:3080
decltype(auto) min(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise minimum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1147
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the implementation of the Submatrix base template.
Header file for the HasSIMDAdd type trait.
Header file for the DenseMatrix base class.
constexpr bool operator>(const NegativeAccuracy< A > &lhs, const T &rhs)
Greater-than comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:370
constexpr bool operator>=(const NegativeAccuracy< A > &, const T &rhs)
Greater-or-equal-than comparison between a NegativeAccuracy object and a floating point value...
Definition: Accuracy.h:446
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:3086
constexpr bool operator==(const NegativeAccuracy< A > &lhs, const T &rhs)
Equality comparison between a NegativeAccuracy object and a floating point value. ...
Definition: Accuracy.h:253
Header file for all SIMD functionality.
constexpr bool Contains_v
Auxiliary variable template for the Contains type trait.The Contains_v variable template provides a c...
Definition: Contains.h:139
Header file for the IsLower type trait.
BLAZE_ALWAYS_INLINE EnableIf_t< IsIntegral_v< T1 > &&HasSize_v< T1, 1UL > > storea(T1 *address, const SIMDi8< T2 > &value) noexcept
Aligned store of a vector of 1-byte integral values.
Definition: Storea.h:78
Constraint on the data type.
Header file for the IsTriangular type trait.
#define BLAZE_CONSTRAINT_MUST_BE_VECTORIZABLE_TYPE(T)
Constraint on the data type.In case the given data type T is not a vectorizable data type...
Definition: Vectorizable.h:61
constexpr bool IsSymmetric_v
Auxiliary variable template for the IsSymmetric type trait.The IsSymmetric_v variable template provid...
Definition: IsSymmetric.h:171
Header file for the exception macros of the math module.
decltype(auto) max(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise maximum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1179
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:438
Header file for the implementation of the SubmatrixData class template.
decltype(auto) operator*(const DenseMatrix< MT1, false > &lhs, const DenseMatrix< MT2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:8908
constexpr bool operator!=(const NegativeAccuracy< A > &lhs, const T &rhs)
Inequality comparison between a NegativeAccuracy object and a floating point value.
Definition: Accuracy.h:293
Constraint on the data type.
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
void clear(const DiagonalProxy< MT > &proxy)
Clearing the represented element.
Definition: DiagonalProxy.h:611
Constraint on the data type.
Flag for aligned vectors and matrices.
Definition: AlignmentFlag.h:65
Header file for the IsSIMDCombinable type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type, a compilation error is created.
Definition: Symmetric.h:79
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for the HasSIMDMult type trait.
Header file for the IsConst type trait.
BLAZE_ALWAYS_INLINE T1 & operator+=(SIMDPack< T1 > &lhs, const SIMDPack< T2 > &rhs)
Addition assignment operator for the addition of two SIMD packs.
Definition: BasicTypes.h:1357
Header file for run time assertion macros.
Header file for the addition trait.
auto smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:131
Header file for the Unique class template.
Header file for the submatrix trait.
Header file for the IsContiguous type trait.
Check< false > Unchecked
Type of the blaze::unchecked instance.blaze::Unchecked is the type of the blaze::unchecked instance...
Definition: Check.h:96
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:133
Header file for the cache size of the target architecture.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:79
Header file for the isDefault shim.
constexpr size_t size(const Matrix< MT, SO > &matrix) noexcept
Returns the total number of elements of the matrix.
Definition: Matrix.h:530
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:100
Constraint on the data type.
Constraint on the data type.
Header file for the HasSIMDSub type trait.
Constraints on the storage order of matrix types.
constexpr bool operator<=(const NegativeAccuracy< A > &, const T &rhs)
Less-or-equal-than comparison between a NegativeAccuracy object and a floating point value...
Definition: Accuracy.h:408
Header file for the HasMutableDataAccess type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
Header file for the IsReference type trait.
auto smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:194
constexpr size_t rows(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of rows of the matrix.
Definition: Matrix.h:498
constexpr bool IsDiagonal_v
Auxiliary variable template for the IsDiagonal type trait.The IsDiagonal_v variable template provides...
Definition: IsDiagonal.h:148
constexpr const DenseIterator< Type, AF > operator+(const DenseIterator< Type, AF > &it, ptrdiff_t inc) noexcept
Addition between a DenseIterator and an integral value.
Definition: DenseIterator.h:718
BLAZE_ALWAYS_INLINE const EnableIf_t< IsIntegral_v< T > &&HasSize_v< T, 1UL >, If_t< IsSigned_v< T >, SIMDint8, SIMDuint8 > > loada(const T *address) noexcept
Loads a vector of 1-byte integral values.
Definition: Loada.h:79
BLAZE_ALWAYS_INLINE EnableIf_t< IsIntegral_v< T1 > &&HasSize_v< T1, 1UL > > stream(T1 *address, const SIMDi8< T2 > &value) noexcept
Aligned, non-temporal store of a vector of 1-byte integral values.
Definition: Stream.h:74
BLAZE_ALWAYS_INLINE bool checkAlignment(const T *address)
Checks the alignment of the given address.
Definition: AlignmentCheck.h:68
const Type & ReturnType
Return type for expression template evaluations.
Definition: CompressedMatrix.h:3081
decltype(auto) trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:765
Flag for unaligned vectors and matrices.
Definition: AlignmentFlag.h:64
auto smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:162
const This & CompositeType
Data type for composite expression templates.
Definition: CompressedMatrix.h:3082
Header file for the alignment check function.
constexpr bool HasSIMDAdd_v
Auxiliary variable template for the HasSIMDAdd type trait.The HasSIMDAdd_v variable template provides...
Definition: HasSIMDAdd.h:188
bool isIntact(const DiagonalMatrix< MT, SO, DF > &m)
Returns whether the invariants of the given diagonal matrix are intact.
Definition: DiagonalMatrix.h:263
bool isDefault(const DiagonalProxy< MT > &proxy)
Returns whether the represented element is in default state.
Definition: DiagonalProxy.h:631
#define BLAZE_THROW_LOGIC_ERROR(MESSAGE)
Macro for the emission of a std::logic_error exception.This macro encapsulates the default way of Bla...
Definition: Exception.h:187
BLAZE_ALWAYS_INLINE T1 & operator-=(SIMDPack< T1 > &lhs, const SIMDPack< T2 > &rhs)
Subtraction assignment operator for the subtraction of two SIMD packs.
Definition: BasicTypes.h:1375
Header file for the IsUpper type trait.
typename DisableIf< Condition, T >::Type DisableIf_t
Auxiliary type for the DisableIf class template.The DisableIf_t alias declaration provides a convenie...
Definition: DisableIf.h:138
Header file for the IsHermitian type trait.
Header file for the IsRestricted type trait.
System settings for the inline keywords.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
Header file for the clear shim.
Header file for the IsExpression type trait class.
Constraint on the data type.
void transpose(Matrix< MT, SO > &matrix)
In-place transpose of the given matrix.
Definition: Matrix.h:825